You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
259 lines
8.9 KiB
Python
259 lines
8.9 KiB
Python
"""Unit tests for libs/backtest/manifests.py."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from libs.backtest.domain import BacktestConfig, ExperimentManifest
|
|
|
|
|
|
REGISTERED_ALIAS_SNAPSHOT_ID = "midlarge-liquid-long-v1_bucketfix_full_audit_tier3"
|
|
REGISTERED_CANONICAL_SNAPSHOT_ID = "midlarge-liquid-long-v1_bucketfix_full_audit_canonical"
|
|
|
|
|
|
def _write_json(path: Path, data: dict) -> None:
|
|
path.write_text(json.dumps(data))
|
|
|
|
|
|
VALID_BASE_CONFIG = {
|
|
"strategy_name": "test_strategy",
|
|
"dataset_snapshot_id": REGISTERED_ALIAS_SNAPSHOT_ID,
|
|
"universe": {
|
|
"min_price": 5.0,
|
|
"min_avg_dollar_volume": 1_000_000,
|
|
"exclude_asset_types": [],
|
|
},
|
|
"signal": {
|
|
"score_threshold": 0.5,
|
|
"max_candidates_per_day": 5,
|
|
"execution_timing": "next_open",
|
|
},
|
|
"risk": {
|
|
"per_trade_risk_pct": 0.01,
|
|
"max_daily_new_risk_pct": 0.03,
|
|
"max_positions": 10,
|
|
"max_positions_per_sector": 3,
|
|
},
|
|
"execution": {
|
|
"entry_fill_model": "next_open",
|
|
"exit_fill_model": "daily_bar_approximation",
|
|
"slippage_bps_base": 10.0,
|
|
"same_bar_priority": "stop_first_conservative",
|
|
},
|
|
"reporting": {
|
|
"write_trade_blotter": True,
|
|
"write_equity_curve": True,
|
|
"write_metrics_summary": True,
|
|
},
|
|
}
|
|
|
|
|
|
class TestLoadBaseConfig:
|
|
def test_load_valid_config(self, tmp_path):
|
|
from libs.backtest.manifests import load_base_config
|
|
|
|
cfg_file = tmp_path / "defaults.json"
|
|
_write_json(cfg_file, VALID_BASE_CONFIG)
|
|
loaded = load_base_config(cfg_file)
|
|
assert loaded["strategy_name"] == "test_strategy"
|
|
|
|
def test_missing_file_raises(self, tmp_path):
|
|
from libs.backtest.manifests import load_base_config
|
|
|
|
with pytest.raises(FileNotFoundError):
|
|
load_base_config(tmp_path / "nonexistent.json")
|
|
|
|
|
|
class TestDeepMerge:
|
|
def test_simple_override(self):
|
|
from libs.backtest.manifests import deep_merge
|
|
|
|
base = {"a": 1, "b": 2}
|
|
overrides = {"b": 99}
|
|
merged = deep_merge(base, overrides)
|
|
assert merged["a"] == 1
|
|
assert merged["b"] == 99
|
|
|
|
def test_nested_merge(self):
|
|
from libs.backtest.manifests import deep_merge
|
|
|
|
base = {"risk": {"max_positions": 10, "per_trade_risk_pct": 0.01}}
|
|
overrides = {"risk": {"max_positions": 5}}
|
|
merged = deep_merge(base, overrides)
|
|
assert merged["risk"]["max_positions"] == 5
|
|
assert merged["risk"]["per_trade_risk_pct"] == 0.01
|
|
|
|
def test_does_not_mutate_base(self):
|
|
from libs.backtest.manifests import deep_merge
|
|
|
|
base = {"a": {"b": 1}}
|
|
overrides = {"a": {"c": 2}}
|
|
deep_merge(base, overrides)
|
|
assert "c" not in base["a"]
|
|
|
|
def test_override_wins(self):
|
|
from libs.backtest.manifests import deep_merge
|
|
|
|
merged = deep_merge({"x": 1}, {"x": 2})
|
|
assert merged["x"] == 2
|
|
|
|
|
|
class TestLoadManifest:
|
|
def test_valid_manifest(self, tmp_path):
|
|
from libs.backtest.manifests import load_manifest
|
|
|
|
manifest_data = {
|
|
"experiment_name": "test_exp",
|
|
"dataset_snapshot_id": REGISTERED_ALIAS_SNAPSHOT_ID,
|
|
"base_config": "configs/backtest/defaults.json",
|
|
"overrides": {},
|
|
}
|
|
f = tmp_path / "manifest.json"
|
|
_write_json(f, manifest_data)
|
|
m = load_manifest(f)
|
|
assert m.experiment_name == "test_exp"
|
|
|
|
def test_missing_file_raises(self, tmp_path):
|
|
from libs.backtest.manifests import load_manifest
|
|
|
|
with pytest.raises(FileNotFoundError):
|
|
load_manifest(tmp_path / "nope.json")
|
|
|
|
|
|
class TestResolveConfig:
|
|
def test_basic_resolve(self, tmp_path):
|
|
from libs.backtest.manifests import load_manifest, resolve_config
|
|
|
|
cfg_file = tmp_path / "defaults.json"
|
|
_write_json(cfg_file, VALID_BASE_CONFIG)
|
|
|
|
manifest_data = {
|
|
"experiment_name": "test",
|
|
"dataset_snapshot_id": REGISTERED_ALIAS_SNAPSHOT_ID,
|
|
"base_config": str(cfg_file),
|
|
"overrides": {},
|
|
}
|
|
m_file = tmp_path / "manifest.json"
|
|
_write_json(m_file, manifest_data)
|
|
manifest = load_manifest(m_file)
|
|
config = resolve_config(manifest)
|
|
assert isinstance(config, BacktestConfig)
|
|
assert config.strategy_name == "test_strategy"
|
|
assert config.dataset_snapshot_id == REGISTERED_CANONICAL_SNAPSHOT_ID
|
|
assert config.requested_snapshot_id == REGISTERED_ALIAS_SNAPSHOT_ID
|
|
assert config.canonical_snapshot_id == REGISTERED_CANONICAL_SNAPSHOT_ID
|
|
|
|
def test_overrides_applied(self, tmp_path):
|
|
from libs.backtest.manifests import load_manifest, resolve_config
|
|
|
|
cfg_file = tmp_path / "defaults.json"
|
|
_write_json(cfg_file, VALID_BASE_CONFIG)
|
|
|
|
manifest_data = {
|
|
"experiment_name": "test",
|
|
"dataset_snapshot_id": REGISTERED_ALIAS_SNAPSHOT_ID,
|
|
"base_config": str(cfg_file),
|
|
"overrides": {"risk": {"max_positions": 3}},
|
|
}
|
|
m_file = tmp_path / "manifest.json"
|
|
_write_json(m_file, manifest_data)
|
|
manifest = load_manifest(m_file)
|
|
config = resolve_config(manifest)
|
|
assert config.risk.max_positions == 3
|
|
assert config.risk.per_trade_risk_pct == 0.01 # from base
|
|
|
|
def test_snapshot_id_override(self, tmp_path):
|
|
from libs.backtest.manifests import load_manifest, resolve_config
|
|
|
|
cfg_file = tmp_path / "defaults.json"
|
|
_write_json(cfg_file, VALID_BASE_CONFIG)
|
|
|
|
manifest_data = {
|
|
"experiment_name": "test",
|
|
"dataset_snapshot_id": REGISTERED_ALIAS_SNAPSHOT_ID,
|
|
"base_config": str(cfg_file),
|
|
"overrides": {},
|
|
}
|
|
m_file = tmp_path / "manifest.json"
|
|
_write_json(m_file, manifest_data)
|
|
manifest = load_manifest(m_file)
|
|
config = resolve_config(
|
|
manifest,
|
|
snapshot_id_override="midlarge-liquid-long-v1_bucketfix_full_audit_tier3tech",
|
|
)
|
|
assert config.dataset_snapshot_id == REGISTERED_CANONICAL_SNAPSHOT_ID
|
|
assert config.requested_snapshot_id == "midlarge-liquid-long-v1_bucketfix_full_audit_tier3tech"
|
|
assert config.canonical_snapshot_id == REGISTERED_CANONICAL_SNAPSHOT_ID
|
|
|
|
def test_strategy_engines_injected_from_manifest(self, tmp_path):
|
|
from libs.backtest.manifests import load_manifest, resolve_config
|
|
|
|
cfg_file = tmp_path / "defaults.json"
|
|
_write_json(cfg_file, VALID_BASE_CONFIG)
|
|
|
|
manifest_data = {
|
|
"experiment_name": "test",
|
|
"dataset_snapshot_id": REGISTERED_ALIAS_SNAPSHOT_ID,
|
|
"base_config": str(cfg_file),
|
|
"overrides": {},
|
|
"strategy_engines": [
|
|
{
|
|
"engine_id": "earnings_same_day_short_v1",
|
|
"event_types": ["earnings_release"],
|
|
"timing_class": "same_day",
|
|
"direction": "short_only",
|
|
"entry_timing_policy": "next_open",
|
|
"max_holding_days": 3,
|
|
"engine_risk_budget_pct": 0.4,
|
|
}
|
|
],
|
|
}
|
|
m_file = tmp_path / "manifest.json"
|
|
_write_json(m_file, manifest_data)
|
|
manifest = load_manifest(m_file)
|
|
config = resolve_config(manifest)
|
|
assert len(config.strategy_engines) == 1
|
|
assert config.strategy_engines[0].engine_id == "earnings_same_day_short_v1"
|
|
|
|
def test_unknown_snapshot_id_raises(self, tmp_path):
|
|
from libs.backtest.manifests import load_manifest, resolve_config
|
|
|
|
cfg_file = tmp_path / "defaults.json"
|
|
_write_json(cfg_file, VALID_BASE_CONFIG)
|
|
|
|
manifest_data = {
|
|
"experiment_name": "test",
|
|
"dataset_snapshot_id": "snap_unknown_for_test",
|
|
"base_config": str(cfg_file),
|
|
"overrides": {},
|
|
}
|
|
m_file = tmp_path / "manifest.json"
|
|
_write_json(m_file, manifest_data)
|
|
manifest = load_manifest(m_file)
|
|
with pytest.raises(ValueError, match="Unknown snapshot id"):
|
|
resolve_config(manifest)
|
|
|
|
|
|
class TestGenerateRunId:
|
|
def test_format(self):
|
|
from libs.backtest.manifests import generate_run_id
|
|
|
|
cfg = BacktestConfig(strategy_name="my_strategy", dataset_snapshot_id="snap_2026_01_01")
|
|
run_id = generate_run_id(cfg)
|
|
assert run_id.startswith("bt_")
|
|
parts = run_id.split("_")
|
|
assert len(parts) >= 4
|
|
|
|
def test_deterministic_for_same_config(self):
|
|
"""Two calls with same config at same time should have same hash suffix."""
|
|
from libs.backtest.manifests import generate_run_id
|
|
|
|
cfg = BacktestConfig(strategy_name="test", dataset_snapshot_id="snap_001")
|
|
id1 = generate_run_id(cfg)
|
|
id2 = generate_run_id(cfg)
|
|
# Hash suffix should be identical
|
|
assert id1.split("_")[-1] == id2.split("_")[-1]
|