fix: SEC EDGAR stale 8-K index auto-refresh + HTTP cache bypass

- fetch_json()에 skip_cache 파라미터 추가 (in-memory/disk 캐시 bypass)
- index_filings()에 force_refresh → skip_cache 자동 연동
- search_filings()에 staleness check (MAX indexed_at vs SEC_DATA_REFRESH_HOURS)
  → 24h 이상 stale한 ticker 자동 re-index (per-ticker asyncio.Lock으로 thundering herd 방지)
- search_filings_bulk()도 동일 staleness 처리
- 테스트 7개 추가 (skip_cache 동작, force_refresh 연동)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

app/services/sec_http_client.py

@@ -180,12 +180,18 @@ class SECHttpClient:
     # HTTP fetch methods
     # ------------------------------------------------------------------
 
-    async def fetch_json(self, url: str) -> dict:
+    async def fetch_json(self, url: str, skip_cache: bool = False) -> dict:
-        """Fetch JSON with retry, backoff, and caching."""
+        """Fetch JSON with retry, backoff, and caching.
+
+        Args:
+            skip_cache: If True, bypass in-memory and disk cache reads (still writes
+                        to cache after a successful fetch so subsequent calls benefit).
+        """
         # In-memory cache
-        if url in self._json_cache:
+        if not skip_cache and url in self._json_cache:
             return self._json_cache[url]
         # Disk cache
+        if not skip_cache:
             try:
                 cp = self._cache_path(url) + ".json"
                 if os.path.exists(cp):

tests/test_sec_filings_fixes.py

@@ -411,3 +411,235 @@ class TestFetchTextMaxBytes:
 
         # Should have tried exactly once (no retry on ValueError)
         assert call_count == 1, f"Expected 1 attempt, got {call_count}"
+
+
+# ---------------------------------------------------------------------------
+# Fix 4: skip_cache in fetch_json
+# ---------------------------------------------------------------------------
+
+class TestFetchJsonSkipCache:
+    """Tests for the skip_cache parameter added to fetch_json."""
+
+    def _make_client(self):
+        from app.services.sec_http_client import SECHttpClient, _TokenBucket
+        client = SECHttpClient.__new__(SECHttpClient)
+        client._pending = 0
+        client._MAX_PENDING = 50
+        client._text_cache = {}
+        client._json_cache = {}
+        client._cache_dir = "/tmp/test_sec_cache_skip"
+        client._req_sem = asyncio.Semaphore(2)
+        client._rate_limiter = _TokenBucket(rate=100.0, capacity=100.0)
+        client.http_timeout = MagicMock()
+        client.http_timeout.total = 12
+        client._session = None
+        return client
+
+    def _make_200_json_session(self, payload: dict):
+        mock_resp = AsyncMock()
+        mock_resp.status = 200
+        mock_resp.json = AsyncMock(return_value=payload)
+        mock_resp.__aenter__ = AsyncMock(return_value=mock_resp)
+        mock_resp.__aexit__ = AsyncMock(return_value=False)
+        mock_session = MagicMock()
+        mock_session.get = MagicMock(return_value=mock_resp)
+        return mock_session
+
+    @pytest.mark.asyncio
+    async def test_skip_cache_false_uses_memory_cache(self):
+        """skip_cache=False (default) returns in-memory cached value without hitting SEC."""
+        client = self._make_client()
+        url = "https://data.sec.gov/submissions/CIK0001234567.json"
+        client._json_cache[url] = {"cached": True}
+
+        call_count = 0
+        async def fake_get_session():
+            nonlocal call_count
+            call_count += 1
+            return MagicMock()
+
+        with patch.object(client, "_get_session", fake_get_session):
+            result = await client.fetch_json(url, skip_cache=False)
+
+        assert result == {"cached": True}
+        assert call_count == 0, "Should serve from memory cache without calling SEC"
+
+    @pytest.mark.asyncio
+    async def test_skip_cache_true_bypasses_memory_cache(self):
+        """skip_cache=True ignores the in-memory cache and fetches fresh from SEC."""
+        client = self._make_client()
+        url = "https://data.sec.gov/submissions/CIK0001234567.json"
+        client._json_cache[url] = {"stale": True}
+
+        fresh_payload = {"fresh": True}
+        mock_session = self._make_200_json_session(fresh_payload)
+
+        with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)):
+            with patch("builtins.open", MagicMock()):
+                with patch("os.path.exists", return_value=False):
+                    result = await client.fetch_json(url, skip_cache=True)
+
+        assert result == fresh_payload, "Should return fresh data ignoring stale cache"
+        # In-memory cache should now be updated with fresh data
+        assert client._json_cache[url] == fresh_payload
+
+    @pytest.mark.asyncio
+    async def test_skip_cache_true_bypasses_disk_cache(self):
+        """skip_cache=True does not read from disk even when disk cache file exists."""
+        client = self._make_client()
+        url = "https://data.sec.gov/submissions/CIK0001234567.json"
+        # Empty in-memory cache to force disk check
+        client._json_cache = {}
+
+        fresh_payload = {"from_sec": True}
+        mock_session = self._make_200_json_session(fresh_payload)
+
+        read_disk_called = False
+        def fake_exists(path):
+            return True  # Pretend disk cache file exists
+
+        original_open = open
+        def fake_open(path, *a, **kw):
+            nonlocal read_disk_called
+            # If this is a READ attempt (mode 'r'), flag it
+            mode = a[0] if a else kw.get('mode', 'r')
+            if 'r' in mode and '.json' in str(path):
+                read_disk_called = True
+            return MagicMock().__enter__.return_value
+
+        with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)):
+            with patch("os.path.exists", fake_exists):
+                with patch("builtins.open", MagicMock()):
+                    # skip_cache=True should skip the disk read block entirely
+                    result = await client.fetch_json(url, skip_cache=True)
+
+        # Disk cache should NOT have been read (we got fresh payload from SEC mock)
+        assert result == fresh_payload
+
+    @pytest.mark.asyncio
+    async def test_skip_cache_false_reads_disk_cache(self):
+        """skip_cache=False (default) reads from disk when in-memory cache is empty."""
+        import json as _json
+        client = self._make_client()
+        url = "https://data.sec.gov/submissions/CIK0001234567.json"
+        client._json_cache = {}  # empty memory cache
+
+        disk_payload = {"from_disk": True}
+
+        import time as _time
+        # Disk cache exists and is fresh
+        with patch("os.path.exists", return_value=True):
+            with patch("os.path.getmtime", return_value=_time.time()):
+                with patch("builtins.open", MagicMock(
+                    return_value=MagicMock(
+                        __enter__=MagicMock(return_value=MagicMock(
+                            read=MagicMock(return_value=_json.dumps(disk_payload))
+                        )),
+                        __exit__=MagicMock(return_value=False)
+                    )
+                )):
+                    with patch("json.load", return_value=disk_payload):
+                        result = await client.fetch_json(url, skip_cache=False)
+
+        assert result == disk_payload
+
+
+# ---------------------------------------------------------------------------
+# Fix 5: index_filings skip_cache / force_refresh -> skip_cache
+# ---------------------------------------------------------------------------
+
+class TestIndexFilingsSkipCache:
+    """Tests that index_filings passes skip_cache correctly to the HTTP client."""
+
+    def _make_service(self):
+        from app.services.sec_filings_service import SECFilingsService
+        svc = SECFilingsService.__new__(SECFilingsService)
+        from app.services.sec_http_client import SECHttpClient
+        svc._http = MagicMock(spec=SECHttpClient)
+        svc._http.sec_base_data = "https://data.sec.gov"
+        svc._http.sec_base_www = "https://www.sec.gov"
+        svc._http.set_deadline = MagicMock()
+        svc._http.clear_deadline = MagicMock()
+        svc._http.remaining_time = MagicMock(return_value=None)
+        svc._doc_fetch_locks = {}
+        svc._reindex_locks = {}
+        svc.SUPPORTED_FORM_TYPES = {"8-K", "10-K"}
+        svc.CHUNK_SIZE = 500
+        return svc
+
+    def _make_empty_submissions(self):
+        return {
+            "filings": {
+                "recent": {
+                    "form": [],
+                    "filingDate": [],
+                    "accessionNumber": [],
+                    "primaryDocument": [],
+                    "primaryDocDescription": [],
+                    "acceptanceDateTime": [],
+                },
+                "files": [],
+            }
+        }
+
+    @pytest.mark.asyncio
+    async def test_force_refresh_implies_skip_cache(self):
+        """force_refresh=True must call fetch_json with skip_cache=True."""
+        svc = self._make_service()
+        svc._http.get_company_cik = AsyncMock(return_value="0001730168")
+        svc._http.fetch_json = AsyncMock(return_value=self._make_empty_submissions())
+
+        mock_db = AsyncMock()
+        mock_db.execute = AsyncMock(return_value=MagicMock(scalars=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[])))))
+
+        await svc.index_filings(mock_db, "AVGO", force_refresh=True)
+
+        # fetch_json must have been called with skip_cache=True
+        calls = svc._http.fetch_json.call_args_list
+        assert len(calls) >= 1
+        for call in calls:
+            _, kwargs = call
+            assert kwargs.get("skip_cache") is True, \
+                f"Expected skip_cache=True when force_refresh=True, got: {call}"
+
+    @pytest.mark.asyncio
+    async def test_skip_cache_false_by_default(self):
+        """Normal index_filings call must NOT bypass cache."""
+        svc = self._make_service()
+        svc._http.get_company_cik = AsyncMock(return_value="0001730168")
+        svc._http.fetch_json = AsyncMock(return_value=self._make_empty_submissions())
+
+        mock_db = AsyncMock()
+        mock_db.execute = AsyncMock(return_value=MagicMock(
+            fetchall=MagicMock(return_value=[])
+        ))
+
+        await svc.index_filings(mock_db, "AVGO")
+
+        calls = svc._http.fetch_json.call_args_list
+        assert len(calls) >= 1
+        for call in calls:
+            _, kwargs = call
+            assert kwargs.get("skip_cache") is False, \
+                f"Expected skip_cache=False by default, got: {call}"
+
+    @pytest.mark.asyncio
+    async def test_explicit_skip_cache_true(self):
+        """Explicit skip_cache=True (without force_refresh) must bypass cache."""
+        svc = self._make_service()
+        svc._http.get_company_cik = AsyncMock(return_value="0001730168")
+        svc._http.fetch_json = AsyncMock(return_value=self._make_empty_submissions())
+
+        mock_db = AsyncMock()
+        mock_db.execute = AsyncMock(return_value=MagicMock(
+            fetchall=MagicMock(return_value=[])
+        ))
+
+        await svc.index_filings(mock_db, "AVGO", skip_cache=True)
+
+        calls = svc._http.fetch_json.call_args_list
+        assert len(calls) >= 1
+        for call in calls:
+            _, kwargs = call
+            assert kwargs.get("skip_cache") is True, \
+                f"Expected skip_cache=True when explicit, got: {call}"