You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
330 lines
11 KiB
Python
330 lines
11 KiB
Python
"""Unit tests for libs/backtest/tracker.py — SQS computation & journal I/O."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from libs.backtest.domain import (
|
|
JournalEntry,
|
|
MetricsBundle,
|
|
SplitResult,
|
|
SQSWeights,
|
|
)
|
|
from libs.backtest.tracker import (
|
|
_normalize,
|
|
_normalize_inverse,
|
|
append_journal_entry,
|
|
build_split_result,
|
|
check_duplicate,
|
|
compute_sqs,
|
|
get_next_entry_id,
|
|
load_journal,
|
|
rebuild_registry,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _normalize / _normalize_inverse
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestNormalize:
|
|
def test_at_low_boundary(self):
|
|
assert _normalize(0.8, low=0.8, high=2.0) == 0.0
|
|
|
|
def test_at_high_boundary(self):
|
|
assert _normalize(2.0, low=0.8, high=2.0) == 100.0
|
|
|
|
def test_midpoint(self):
|
|
assert _normalize(1.4, low=0.8, high=2.0) == pytest.approx(50.0)
|
|
|
|
def test_below_low_clamps(self):
|
|
assert _normalize(0.0, low=0.8, high=2.0) == 0.0
|
|
|
|
def test_above_high_clamps(self):
|
|
assert _normalize(5.0, low=0.8, high=2.0) == 100.0
|
|
|
|
def test_none_returns_zero(self):
|
|
assert _normalize(None, low=0.8, high=2.0) == 0.0
|
|
|
|
|
|
class TestNormalizeInverse:
|
|
def test_dd_at_worst(self):
|
|
# 10% drawdown = worst (0 pts)
|
|
assert _normalize_inverse(10.0, low=10.0, high=1.0) == 0.0
|
|
|
|
def test_dd_at_best(self):
|
|
# 1% drawdown = best (100 pts)
|
|
assert _normalize_inverse(1.0, low=10.0, high=1.0) == 100.0
|
|
|
|
def test_dd_midpoint(self):
|
|
assert _normalize_inverse(5.5, low=10.0, high=1.0) == pytest.approx(50.0)
|
|
|
|
def test_none_returns_zero(self):
|
|
assert _normalize_inverse(None, low=10.0, high=1.0) == 0.0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# compute_sqs
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestComputeSQS:
|
|
def test_perfect_metrics(self):
|
|
"""All metrics at 100-point boundaries => SQS near 100."""
|
|
m = MetricsBundle(
|
|
trade_count=200,
|
|
profit_factor=2.0,
|
|
total_return_pct=5.0,
|
|
max_drawdown_pct=1.0,
|
|
sharpe_ratio=2.0,
|
|
win_rate=0.65,
|
|
monthly_win_rate=0.70,
|
|
equity_curve_r_squared=0.80,
|
|
)
|
|
sqs, breakdown = compute_sqs(m)
|
|
assert sqs == pytest.approx(100.0, abs=0.5)
|
|
assert breakdown["profitability"] == pytest.approx(100.0, abs=0.5)
|
|
assert breakdown["risk"] == pytest.approx(100.0, abs=0.5)
|
|
assert breakdown["consistency"] == pytest.approx(100.0, abs=0.5)
|
|
assert breakdown["robustness"] == pytest.approx(100.0, abs=0.5)
|
|
|
|
def test_worst_metrics(self):
|
|
"""All metrics at 0-point boundaries => SQS = 0."""
|
|
m = MetricsBundle(
|
|
trade_count=5,
|
|
profit_factor=0.5,
|
|
total_return_pct=-10.0,
|
|
max_drawdown_pct=15.0,
|
|
sharpe_ratio=-2.0,
|
|
win_rate=0.20,
|
|
monthly_win_rate=0.10,
|
|
equity_curve_r_squared=-0.5,
|
|
)
|
|
sqs, _ = compute_sqs(m)
|
|
assert sqs == 0.0
|
|
|
|
def test_low_trade_penalty(self):
|
|
"""< 20 trades => SQS * 0.5."""
|
|
m = MetricsBundle(
|
|
trade_count=15,
|
|
profit_factor=1.5,
|
|
total_return_pct=2.0,
|
|
max_drawdown_pct=3.0,
|
|
sharpe_ratio=1.0,
|
|
win_rate=0.55,
|
|
monthly_win_rate=0.55,
|
|
equity_curve_r_squared=0.5,
|
|
)
|
|
sqs_penalized, _ = compute_sqs(m)
|
|
|
|
m_enough = m.model_copy(update={"trade_count": 100})
|
|
sqs_full, _ = compute_sqs(m_enough)
|
|
|
|
# Penalised score should be roughly half (trade_count affects robustness sub-score too)
|
|
assert sqs_penalized < sqs_full
|
|
assert sqs_penalized > 0
|
|
|
|
def test_midrange_metrics(self):
|
|
"""Typical mid-range strategy should score 30-60."""
|
|
m = MetricsBundle(
|
|
trade_count=80,
|
|
profit_factor=1.1,
|
|
total_return_pct=0.5,
|
|
max_drawdown_pct=5.0,
|
|
sharpe_ratio=0.5,
|
|
win_rate=0.50,
|
|
monthly_win_rate=0.50,
|
|
equity_curve_r_squared=0.30,
|
|
)
|
|
sqs, _ = compute_sqs(m)
|
|
assert 30 <= sqs <= 65
|
|
|
|
def test_custom_weights(self):
|
|
"""Custom weights should change the SQS."""
|
|
m = MetricsBundle(
|
|
trade_count=80,
|
|
profit_factor=2.0,
|
|
total_return_pct=5.0,
|
|
max_drawdown_pct=8.0,
|
|
sharpe_ratio=0.0,
|
|
win_rate=0.40,
|
|
monthly_win_rate=0.40,
|
|
)
|
|
w_profit_heavy = SQSWeights(profitability=0.80, risk=0.10, consistency=0.05, robustness=0.05)
|
|
w_risk_heavy = SQSWeights(profitability=0.10, risk=0.80, consistency=0.05, robustness=0.05)
|
|
|
|
sqs_profit, _ = compute_sqs(m, w_profit_heavy)
|
|
sqs_risk, _ = compute_sqs(m, w_risk_heavy)
|
|
# This strategy has great profitability but mediocre risk
|
|
assert sqs_profit > sqs_risk
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# build_split_result
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_build_split_result():
|
|
m = MetricsBundle(
|
|
trade_count=50,
|
|
profit_factor=1.2,
|
|
total_return_pct=2.5,
|
|
win_rate=0.55,
|
|
max_drawdown_pct=3.0,
|
|
sharpe_ratio=0.8,
|
|
monthly_win_rate=0.60,
|
|
equity_curve_r_squared=0.40,
|
|
)
|
|
sr = build_split_result("test", "bt_run123", m)
|
|
assert sr.run_id == "bt_run123"
|
|
assert sr.trade_count == 50
|
|
assert sr.profit_factor == 1.2
|
|
assert sr.total_return_pct == 2.5
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Journal I/O
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestJournalIO:
|
|
def test_append_and_load(self, tmp_path):
|
|
journal_path = tmp_path / "journal.jsonl"
|
|
entry = JournalEntry(
|
|
entry_id="IMP-0001",
|
|
timestamp="2026-03-16T12:00:00",
|
|
experiment_name="test_exp_1",
|
|
hypothesis="Test hypothesis",
|
|
sqs_score=55.0,
|
|
sqs_breakdown={"profitability": 60.0, "risk": 50.0, "consistency": 55.0, "robustness": 50.0},
|
|
verdict="better",
|
|
)
|
|
append_journal_entry(journal_path, entry)
|
|
|
|
entries = load_journal(journal_path)
|
|
assert len(entries) == 1
|
|
assert entries[0].entry_id == "IMP-0001"
|
|
assert entries[0].experiment_name == "test_exp_1"
|
|
assert entries[0].sqs_score == 55.0
|
|
|
|
def test_multiple_entries(self, tmp_path):
|
|
journal_path = tmp_path / "journal.jsonl"
|
|
for i in range(3):
|
|
entry = JournalEntry(
|
|
entry_id=f"IMP-{i+1:04d}",
|
|
timestamp=f"2026-03-{16+i}T12:00:00",
|
|
experiment_name=f"exp_{i}",
|
|
hypothesis=f"Hypothesis {i}",
|
|
sqs_score=float(40 + i * 10),
|
|
)
|
|
append_journal_entry(journal_path, entry)
|
|
|
|
entries = load_journal(journal_path)
|
|
assert len(entries) == 3
|
|
assert entries[2].sqs_score == 60.0
|
|
|
|
def test_get_next_entry_id(self, tmp_path):
|
|
journal_path = tmp_path / "journal.jsonl"
|
|
assert get_next_entry_id(journal_path) == "IMP-0001"
|
|
|
|
entry = JournalEntry(
|
|
entry_id="IMP-0001",
|
|
timestamp="2026-03-16T12:00:00",
|
|
experiment_name="exp_1",
|
|
hypothesis="h",
|
|
)
|
|
append_journal_entry(journal_path, entry)
|
|
assert get_next_entry_id(journal_path) == "IMP-0002"
|
|
|
|
def test_load_empty(self, tmp_path):
|
|
journal_path = tmp_path / "nonexistent.jsonl"
|
|
entries = load_journal(journal_path)
|
|
assert entries == []
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# check_duplicate
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestCheckDuplicate:
|
|
def test_finds_duplicates(self, tmp_path):
|
|
journal_path = tmp_path / "journal.jsonl"
|
|
for name in ["exp_a", "exp_b", "exp_a"]:
|
|
entry = JournalEntry(
|
|
entry_id=get_next_entry_id(journal_path),
|
|
timestamp="2026-03-16T12:00:00",
|
|
experiment_name=name,
|
|
hypothesis="h",
|
|
)
|
|
append_journal_entry(journal_path, entry)
|
|
|
|
dupes = check_duplicate(journal_path, "exp_a")
|
|
assert len(dupes) == 2
|
|
|
|
def test_no_duplicates(self, tmp_path):
|
|
journal_path = tmp_path / "journal.jsonl"
|
|
entry = JournalEntry(
|
|
entry_id="IMP-0001",
|
|
timestamp="2026-03-16T12:00:00",
|
|
experiment_name="exp_a",
|
|
hypothesis="h",
|
|
)
|
|
append_journal_entry(journal_path, entry)
|
|
|
|
dupes = check_duplicate(journal_path, "exp_z")
|
|
assert len(dupes) == 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# rebuild_registry
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestRebuildRegistry:
|
|
def test_registry_and_leaderboard(self, tmp_path):
|
|
journal_path = tmp_path / "journal.jsonl"
|
|
registry_path = tmp_path / "registry.json"
|
|
leaderboard_path = tmp_path / "LEADERBOARD.md"
|
|
|
|
# Create entries with different SQS scores
|
|
for i, (name, sqs) in enumerate([("exp_low", 30.0), ("exp_high", 70.0), ("exp_mid", 50.0)]):
|
|
test_result = SplitResult(
|
|
run_id=f"bt_{name}",
|
|
trade_count=60,
|
|
profit_factor=1.0 + i * 0.2,
|
|
total_return_pct=float(i),
|
|
win_rate=0.5,
|
|
)
|
|
entry = JournalEntry(
|
|
entry_id=f"IMP-{i+1:04d}",
|
|
timestamp=f"2026-03-{16+i}T12:00:00",
|
|
experiment_name=name,
|
|
hypothesis=f"h{i}",
|
|
sqs_score=sqs,
|
|
results={"test": test_result},
|
|
verdict="better" if sqs > 50 else "worse",
|
|
)
|
|
append_journal_entry(journal_path, entry)
|
|
|
|
registry = rebuild_registry(journal_path, registry_path, leaderboard_path)
|
|
|
|
# Sorted by SQS descending
|
|
assert len(registry.entries) == 3
|
|
assert registry.entries[0].experiment_name == "exp_high"
|
|
assert registry.entries[0].sqs_score == 70.0
|
|
assert registry.entries[2].experiment_name == "exp_low"
|
|
|
|
# Files exist
|
|
assert registry_path.exists()
|
|
assert leaderboard_path.exists()
|
|
|
|
# Leaderboard contains table
|
|
lb_text = leaderboard_path.read_text()
|
|
assert "exp_high" in lb_text
|
|
assert "exp_low" in lb_text
|
|
assert "| # |" in lb_text
|