You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
114 lines
3.6 KiB
Python
114 lines
3.6 KiB
Python
"""Replay test: verify same document → LLM cache hit on second call."""
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
|
|
import pytest
|
|
|
|
pytestmark = pytest.mark.replay
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_same_document_hits_cache_on_replay() -> None:
|
|
"""Processing the same document twice should hit cache on the second call."""
|
|
from libs.llm.cache import LLMCacheStore
|
|
from libs.llm.client import OllamaClient
|
|
from libs.llm.parser import LLMParser
|
|
|
|
# Simulate cache miss on first call, hit on second
|
|
cached_output = {
|
|
"schema_version": "1.0.0",
|
|
"document_id": "DOC::replay_test",
|
|
"parser_kind": "llm",
|
|
"event_type": "earnings_release",
|
|
"event_direction": "bullish",
|
|
"event_date": "2026-01-15",
|
|
"filing_time_bucket": "post_market",
|
|
"headline": "Q1 results beat",
|
|
"summary": "Earnings beat estimates.",
|
|
"guidance": {"status": "raised", "scope": "annual", "notes": ""},
|
|
"signals": {
|
|
"demand_strength": "strong",
|
|
"pricing_power": "present",
|
|
"backlog_or_bookings": "present",
|
|
"customer_expansion": "present",
|
|
"margin_quality": "improving",
|
|
},
|
|
"risk_flags": {
|
|
"oneoff_item": False,
|
|
"tax_benefit": False,
|
|
"valuation_gain": False,
|
|
"non_gaap_heavy": False,
|
|
"financing_related": False,
|
|
"legal_or_regulatory_overhang": False,
|
|
},
|
|
"evidence": [],
|
|
"confidence": {
|
|
"overall": 0.85,
|
|
"event_type": 0.90,
|
|
"event_direction": 0.80,
|
|
"guidance": 0.75,
|
|
"risk_flags": 0.90,
|
|
},
|
|
"warnings": [],
|
|
}
|
|
|
|
call_count = {"n": 0}
|
|
|
|
async def mock_get(session: object, cache_key: str) -> dict | None:
|
|
if call_count["n"] == 0:
|
|
return None # cache miss on first call
|
|
return cached_output # cache hit on subsequent calls
|
|
|
|
async def mock_put(*args: object, **kwargs: object) -> None:
|
|
call_count["n"] += 1 # increment after first call stores to cache
|
|
|
|
mock_cache = MagicMock(spec=LLMCacheStore)
|
|
mock_cache.get = AsyncMock(side_effect=mock_get)
|
|
mock_cache.put = AsyncMock(side_effect=mock_put)
|
|
|
|
# Mock Ollama client (only called once)
|
|
mock_ollama = MagicMock(spec=OllamaClient)
|
|
mock_ollama.model = "llama3.2"
|
|
mock_ollama.chat = AsyncMock(
|
|
return_value=(
|
|
{k: v for k, v in cached_output.items() if k != "schema_version"},
|
|
{"prompt_tokens": 100, "completion_tokens": 50},
|
|
300,
|
|
)
|
|
)
|
|
|
|
llm_parser = LLMParser(client=mock_ollama, cache_store=mock_cache)
|
|
|
|
doc_text = "Apple reports Q1 earnings: revenue $123B, EPS $2.50, beats estimates."
|
|
doc_meta = {
|
|
"form_type": "8-K",
|
|
"filing_date": "2026-01-15",
|
|
"filing_time_bucket": "post_market",
|
|
}
|
|
|
|
# First call — cache miss, Ollama called
|
|
mock_session1 = AsyncMock()
|
|
result1 = await llm_parser.parse(
|
|
document_id="DOC::replay_test",
|
|
doc_text=doc_text,
|
|
doc_meta=doc_meta,
|
|
rule_hints={},
|
|
session=mock_session1,
|
|
)
|
|
assert result1 is not None
|
|
assert mock_ollama.chat.call_count == 1
|
|
|
|
# Second call — cache hit, Ollama NOT called again
|
|
mock_session2 = AsyncMock()
|
|
result2 = await llm_parser.parse(
|
|
document_id="DOC::replay_test",
|
|
doc_text=doc_text,
|
|
doc_meta=doc_meta,
|
|
rule_hints={},
|
|
session=mock_session2,
|
|
)
|
|
assert result2 == cached_output
|
|
# Ollama was still only called once (not twice)
|
|
assert mock_ollama.chat.call_count == 1
|