You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
204 lines
7.2 KiB
Python
204 lines
7.2 KiB
Python
"""Replay test: same fixture → same parser output (determinism)."""
|
|
import pytest
|
|
|
|
SAMPLE_TEXT = """
|
|
Item 2.02 Results of Operations
|
|
|
|
Apple today announced record quarterly revenue of $124.3 billion.
|
|
Guidance raised for Q2. Demand remains strong. Margin expansion driven by Services.
|
|
Customer additions in enterprise continue. Adjusted EPS excludes non-GAAP items.
|
|
"""
|
|
|
|
|
|
@pytest.mark.replay
|
|
def test_parser_determinism():
|
|
"""Running parser twice on same text yields identical output."""
|
|
from libs.parser.rule_parser import RuleBasedParser
|
|
|
|
p = RuleBasedParser()
|
|
metadata = {"filing_date": "2026-01-29", "accepted_at_utc": "2026-01-29T21:05:00Z"}
|
|
|
|
out1 = p.parse("DOC::test", "8-K", SAMPLE_TEXT, metadata)
|
|
out2 = p.parse("DOC::test", "8-K", SAMPLE_TEXT, metadata)
|
|
|
|
assert out1.event_type == out2.event_type
|
|
assert out1.event_direction == out2.event_direction
|
|
assert out1.guidance.status == out2.guidance.status
|
|
assert out1.confidence.overall == out2.confidence.overall
|
|
assert out1.filing_time_bucket == out2.filing_time_bucket
|
|
|
|
|
|
@pytest.mark.replay
|
|
def test_parser_determinism_negative_text():
|
|
"""Determinism holds for negative/mixed text too."""
|
|
from libs.parser.rule_parser import RuleBasedParser
|
|
|
|
negative_text = """
|
|
Item 2.02 Results of Operations
|
|
Revenue below expectations. Guidance lowered. Demand softness observed.
|
|
Convertible note offering announced. Margin compression continues.
|
|
"""
|
|
p = RuleBasedParser()
|
|
metadata = {"filing_date": "2026-02-01"}
|
|
out1 = p.parse("DOC::test2", "8-K", negative_text, metadata)
|
|
out2 = p.parse("DOC::test2", "8-K", negative_text, metadata)
|
|
assert out1.event_type == out2.event_type
|
|
assert out1.guidance.status == out2.guidance.status
|
|
|
|
|
|
@pytest.mark.replay
|
|
def test_feature_determinism(sample_parser_output):
|
|
"""Event features are deterministic for same parser output."""
|
|
from libs.features.event_features import compute_event_features
|
|
|
|
f1 = compute_event_features(sample_parser_output)
|
|
f2 = compute_event_features(sample_parser_output)
|
|
assert f1 == f2
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Backtest determinism tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _build_synthetic_store():
|
|
"""Reusable synthetic SnapshotStore for backtest replay tests."""
|
|
import datetime as dt
|
|
from zoneinfo import ZoneInfo
|
|
|
|
from libs.backtest.snapshot_store import SnapshotStore
|
|
|
|
candidates = {
|
|
dt.date(2026, 1, 5): [
|
|
{
|
|
"event_id": "EVT::REPLAY::001",
|
|
"symbol": "AAPL",
|
|
"execution_date": dt.date(2026, 1, 5),
|
|
"entry_date": "2026-01-05",
|
|
"entry_price": 150.0,
|
|
"score": 0.85,
|
|
"sector": "Technology",
|
|
"event_type": "earnings",
|
|
"event_timestamp": "2026-01-02T21:00:00+00:00",
|
|
"filing_time_bucket": "post_market",
|
|
"reaction_date": "2026-01-02",
|
|
"avg_dollar_volume": 5_000_000.0,
|
|
"atr_14": 3.0,
|
|
}
|
|
],
|
|
}
|
|
bars = {
|
|
"AAPL": {
|
|
dt.date(2026, 1, 5): {
|
|
"date": dt.date(2026, 1, 5),
|
|
"open": 150.0, "high": 160.0, "low": 148.0, "close": 157.0, "volume": 1_000_000,
|
|
},
|
|
dt.date(2026, 1, 6): {
|
|
"date": dt.date(2026, 1, 6),
|
|
"open": 157.0, "high": 168.0, "low": 155.0, "close": 165.0, "volume": 900_000,
|
|
},
|
|
}
|
|
}
|
|
return SnapshotStore(candidates_by_exec_date=candidates, bars_by_symbol_date=bars)
|
|
|
|
|
|
def _make_backtest_config():
|
|
from libs.backtest.domain import (
|
|
BacktestConfig,
|
|
ExecutionConfig,
|
|
ReportingConfig,
|
|
RiskConfig,
|
|
SignalConfig,
|
|
UniverseConfig,
|
|
)
|
|
|
|
return BacktestConfig(
|
|
strategy_name="replay_test",
|
|
dataset_snapshot_id="test_snapshot",
|
|
universe=UniverseConfig(min_price=5.0, min_avg_dollar_volume=100_000),
|
|
signal=SignalConfig(score_threshold=0.5, max_candidates_per_day=5),
|
|
risk=RiskConfig(
|
|
per_trade_risk_pct=0.01,
|
|
max_daily_new_risk_pct=0.05,
|
|
max_positions=10,
|
|
max_positions_per_sector=5,
|
|
),
|
|
execution=ExecutionConfig(
|
|
entry_fill_model="next_open",
|
|
exit_fill_model="daily_bar_approximation",
|
|
slippage_bps_base=10.0,
|
|
commission_per_share=0.005,
|
|
same_bar_priority="stop_first_conservative",
|
|
max_holding_days=10,
|
|
),
|
|
reporting=ReportingConfig(
|
|
write_trade_blotter=False,
|
|
write_equity_curve=False,
|
|
write_metrics_summary=False,
|
|
),
|
|
)
|
|
|
|
|
|
@pytest.mark.replay
|
|
def test_backtest_determinism():
|
|
"""Running the backtest twice on the same synthetic store yields identical metrics."""
|
|
from apps.backtester.run import BacktestRunner
|
|
from libs.backtest.domain import ExperimentManifest
|
|
|
|
manifest = ExperimentManifest(
|
|
experiment_name="replay_test",
|
|
dataset_snapshot_id="test_snapshot",
|
|
base_config="configs/backtest/defaults.json",
|
|
overrides={},
|
|
)
|
|
config = _make_backtest_config()
|
|
|
|
store1 = _build_synthetic_store()
|
|
runner1 = BacktestRunner(manifest=manifest, config=config, store=store1, initial_equity=100_000.0)
|
|
result1 = runner1.run()
|
|
|
|
store2 = _build_synthetic_store()
|
|
runner2 = BacktestRunner(manifest=manifest, config=config, store=store2, initial_equity=100_000.0)
|
|
result2 = runner2.run()
|
|
|
|
# Core metrics must be identical
|
|
assert result1.metrics.trade_count == result2.metrics.trade_count
|
|
assert result1.metrics.win_rate == result2.metrics.win_rate
|
|
assert result1.metrics.total_return_pct == result2.metrics.total_return_pct
|
|
assert result1.metrics.max_drawdown_pct == result2.metrics.max_drawdown_pct
|
|
assert result1.total_candidates_seen == result2.total_candidates_seen
|
|
assert result1.total_orders_rejected == result2.total_orders_rejected
|
|
assert result1.total_trading_days == result2.total_trading_days
|
|
|
|
|
|
@pytest.mark.replay
|
|
def test_backtest_selector_determinism():
|
|
"""Selector ranking is deterministic across multiple calls."""
|
|
from libs.backtest.domain import SignalConfig, UniverseConfig
|
|
from libs.backtest.selector import select_candidates
|
|
|
|
rows = [
|
|
{
|
|
"event_id": f"EVT::{'ABCDE'[i]}",
|
|
"symbol": "ABCDE"[i],
|
|
"entry_date": "2026-01-05",
|
|
"entry_price": 100.0 + i,
|
|
"score": 0.9 - i * 0.05,
|
|
"sector": "Technology",
|
|
"event_type": "earnings",
|
|
"event_timestamp": "2026-01-02T21:00:00+00:00",
|
|
"filing_time_bucket": "post_market",
|
|
"reaction_date": "2026-01-02",
|
|
"avg_dollar_volume": 5_000_000.0 + i * 100_000,
|
|
"atr_14": 2.0,
|
|
}
|
|
for i in range(5)
|
|
]
|
|
u = UniverseConfig(min_price=5.0, min_avg_dollar_volume=100_000)
|
|
s = SignalConfig(score_threshold=0.5, max_candidates_per_day=10)
|
|
|
|
result1 = select_candidates(rows, u, s)
|
|
result2 = select_candidates(rows, u, s)
|
|
|
|
assert [c.symbol for c in result1] == [c.symbol for c in result2]
|