"""Unit tests for libs/backtest/tracker.py — SQS computation & journal I/O.""" from __future__ import annotations import datetime as dt import json import multiprocessing import time from pathlib import Path import pytest from libs.backtest.domain import ( CommonWindowSummary, JournalEntry, MetricsBundle, MultiCapitalCommonWindowSummary, ResetCommonWindowSummary, RobustnessHorizonSummary, RobustnessMatrixSummary, SplitResult, SQSWeights, WalkForwardAggregate, WalkForwardFoldResult, WalkForwardGapStats, WalkForwardSummary, ) from libs.backtest.tracker import ( _normalize, _normalize_band, _normalize_inverse, _gap_penalty, _overfitting_penalty, _fold_variance_penalty, _trade_credibility, _engine_reliability_penalty, attach_out_of_time_robustness_summary, attach_robustness_summary, attach_walk_forward_summary, append_journal_entry, build_split_result, check_duplicate, classify_strategy_family, compute_deployment_score, compute_public_sqs, compute_public_sqs_v2, compute_public_sqs_v3, compute_public_sqs_v4, compute_public_sqs_v5, compute_public_sqs_v6, compute_public_sqs_v7, compute_public_sqs_v8, compute_public_sqs_v9, _resolve_regime_score, compute_oot_factor_v2, compute_common_window_score, compute_multi_capital_common_window_score, compute_reset_common_window_score, compute_promotion_score, _compute_oot_positive_rate_63plus, compute_oot_robustness_gate, compute_oot_robustness_quality, compute_robustness_gate, compute_rqs, compute_sqs, compute_sqs_v2, compute_unified_score, compute_unified_split_quality, compute_wfqs, compute_wfqs_v2, filter_registry_entries, get_next_entry_id, is_active_research_entry, journal_lock, load_journal, refresh_public_scores, rebuild_registry, scan_runs_for_experiment, sync_official_manifests, ) def _write_locked_journal_entry(payload: tuple[str, str]) -> str: journal_path_str, experiment_name = payload journal_path = Path(journal_path_str) with journal_lock(journal_path): entry_id = get_next_entry_id(journal_path) time.sleep(0.05) append_journal_entry( journal_path, JournalEntry( entry_id=entry_id, timestamp="2026-03-17T10:10:07+00:00", experiment_name=experiment_name, hypothesis="h", ), ) return entry_id # --------------------------------------------------------------------------- # _normalize / _normalize_inverse # --------------------------------------------------------------------------- class TestNormalize: def test_at_low_boundary(self): assert _normalize(0.8, low=0.8, high=2.0) == 0.0 def test_at_high_boundary(self): assert _normalize(2.0, low=0.8, high=2.0) == 100.0 def test_midpoint(self): assert _normalize(1.4, low=0.8, high=2.0) == pytest.approx(50.0) def test_below_low_clamps(self): assert _normalize(0.0, low=0.8, high=2.0) == 0.0 def test_above_high_clamps(self): assert _normalize(5.0, low=0.8, high=2.0) == 100.0 def test_none_returns_zero(self): assert _normalize(None, low=0.8, high=2.0) == 0.0 class TestNormalizeInverse: def test_dd_at_worst(self): # 10% drawdown = worst (0 pts) assert _normalize_inverse(10.0, low=10.0, high=1.0) == 0.0 def test_dd_at_best(self): # 1% drawdown = best (100 pts) assert _normalize_inverse(1.0, low=10.0, high=1.0) == 100.0 def test_dd_midpoint(self): assert _normalize_inverse(5.5, low=10.0, high=1.0) == pytest.approx(50.0) def test_none_returns_zero(self): assert _normalize_inverse(None, low=10.0, high=1.0) == 0.0 class TestNormalizeBand: def test_band_plateau_scores_max(self): assert _normalize_band(60.0, low_bad=10.0, low_good=40.0, high_good=80.0, high_bad=100.0) == 100.0 def test_below_band_ramps_up(self): assert _normalize_band(25.0, low_bad=10.0, low_good=40.0, high_good=80.0, high_bad=100.0) == pytest.approx(50.0) def test_outside_band_scores_zero(self): assert _normalize_band(5.0, low_bad=10.0, low_good=40.0, high_good=80.0, high_bad=100.0) == 0.0 assert _normalize_band(100.0, low_bad=10.0, low_good=40.0, high_good=80.0, high_bad=100.0) == 0.0 # --------------------------------------------------------------------------- # compute_sqs # --------------------------------------------------------------------------- class TestComputeSQS: def test_perfect_metrics(self): """All metrics at 100-point boundaries => SQS near 100.""" m = MetricsBundle( trade_count=200, profit_factor=2.0, total_return_pct=5.0, max_drawdown_pct=1.0, sharpe_ratio=2.0, win_rate=0.65, monthly_win_rate=0.70, equity_curve_r_squared=0.80, ) sqs, breakdown = compute_sqs(m) assert sqs == pytest.approx(100.0, abs=0.5) assert breakdown["profitability"] == pytest.approx(100.0, abs=0.5) assert breakdown["risk"] == pytest.approx(100.0, abs=0.5) assert breakdown["consistency"] == pytest.approx(100.0, abs=0.5) assert breakdown["robustness"] == pytest.approx(100.0, abs=0.5) def test_worst_metrics(self): """All metrics at 0-point boundaries => SQS = 0.""" m = MetricsBundle( trade_count=5, profit_factor=0.5, total_return_pct=-10.0, max_drawdown_pct=15.0, sharpe_ratio=-2.0, win_rate=0.20, monthly_win_rate=0.10, equity_curve_r_squared=-0.5, ) sqs, _ = compute_sqs(m) assert sqs == 0.0 def test_low_trade_penalty(self): """< 20 trades => SQS * 0.5.""" m = MetricsBundle( trade_count=15, profit_factor=1.5, total_return_pct=2.0, max_drawdown_pct=3.0, sharpe_ratio=1.0, win_rate=0.55, monthly_win_rate=0.55, equity_curve_r_squared=0.5, ) sqs_penalized, _ = compute_sqs(m) m_enough = m.model_copy(update={"trade_count": 100}) sqs_full, _ = compute_sqs(m_enough) # Penalised score should be roughly half (trade_count affects robustness sub-score too) assert sqs_penalized < sqs_full assert sqs_penalized > 0 def test_midrange_metrics(self): """Typical mid-range strategy should score 30-60.""" m = MetricsBundle( trade_count=80, profit_factor=1.1, total_return_pct=0.5, max_drawdown_pct=5.0, sharpe_ratio=0.5, win_rate=0.50, monthly_win_rate=0.50, equity_curve_r_squared=0.30, ) sqs, _ = compute_sqs(m) assert 30 <= sqs <= 65 def test_custom_weights(self): """Custom weights should change the SQS.""" m = MetricsBundle( trade_count=80, profit_factor=2.0, total_return_pct=5.0, max_drawdown_pct=8.0, sharpe_ratio=0.0, win_rate=0.40, monthly_win_rate=0.40, ) w_profit_heavy = SQSWeights(profitability=0.80, risk=0.10, consistency=0.05, robustness=0.05) w_risk_heavy = SQSWeights(profitability=0.10, risk=0.80, consistency=0.05, robustness=0.05) sqs_profit, _ = compute_sqs(m, w_profit_heavy) sqs_risk, _ = compute_sqs(m, w_risk_heavy) # This strategy has great profitability but mediocre risk assert sqs_profit > sqs_risk class TestComputeSQSv2: def test_missing_exposure_returns_none(self): m = MetricsBundle( trade_count=50, profit_factor=1.5, total_return_pct=2.0, max_drawdown_pct=2.0, sharpe_ratio=1.2, win_rate=0.55, monthly_win_rate=0.60, equity_curve_r_squared=0.50, ) sqs_v2, breakdown = compute_sqs_v2(m) assert sqs_v2 is None assert breakdown == {} def test_capital_efficiency_can_break_tie(self): base = dict( trade_count=50, profit_factor=1.5, total_return_pct=2.0, max_drawdown_pct=2.0, sharpe_ratio=1.2, win_rate=0.55, monthly_win_rate=0.60, equity_curve_r_squared=0.50, days_in_market_pct=60.0, ) efficient = MetricsBundle( **base, avg_gross_exposure_pct=4.0, avg_net_exposure_pct=-1.0, ) inefficient = MetricsBundle( **base, avg_gross_exposure_pct=10.0, avg_net_exposure_pct=-1.0, ) efficient_score, _ = compute_sqs_v2(efficient) inefficient_score, _ = compute_sqs_v2(inefficient) assert efficient_score is not None assert inefficient_score is not None assert efficient_score > inefficient_score class TestComputePromotionScore: def test_requires_valid_and_test(self): score, breakdown = compute_promotion_score(None, None) assert score is None assert breakdown == {} def test_penalizes_test_only_outperformance(self): overfit_test = SplitResult( run_id="bt_test", trade_count=56, profit_factor=2.0, total_return_pct=2.2, win_rate=0.61, max_drawdown_pct=0.4, sharpe_ratio=4.2, monthly_win_rate=1.0, equity_curve_r_squared=0.88, avg_gross_exposure_pct=6.2, avg_net_exposure_pct=1.1, days_in_market_pct=76.6, ) overfit_valid = SplitResult( run_id="bt_valid", trade_count=53, profit_factor=1.24, total_return_pct=0.7, win_rate=0.51, max_drawdown_pct=0.9, sharpe_ratio=1.2, monthly_win_rate=0.75, equity_curve_r_squared=0.15, avg_gross_exposure_pct=7.5, avg_net_exposure_pct=1.6, days_in_market_pct=77.2, ) robust_test = SplitResult( run_id="bt_test_robust", trade_count=22, profit_factor=4.19, total_return_pct=1.24, win_rate=0.73, max_drawdown_pct=0.22, sharpe_ratio=3.6, monthly_win_rate=0.67, equity_curve_r_squared=0.92, avg_gross_exposure_pct=2.3, avg_net_exposure_pct=-0.8, days_in_market_pct=42.6, ) robust_valid = SplitResult( run_id="bt_valid_robust", trade_count=28, profit_factor=2.31, total_return_pct=1.3, win_rate=0.68, max_drawdown_pct=0.37, sharpe_ratio=3.05, monthly_win_rate=0.75, equity_curve_r_squared=0.49, avg_gross_exposure_pct=4.1, avg_net_exposure_pct=-1.9, days_in_market_pct=57.9, ) overfit_score, overfit_breakdown = compute_promotion_score(overfit_test, overfit_valid) robust_score, robust_breakdown = compute_promotion_score(robust_test, robust_valid) assert overfit_score is not None assert robust_score is not None assert robust_score > overfit_score assert overfit_breakdown["floor_quality"] < overfit_breakdown["test_quality"] class TestComputeUnifiedScore: def test_requires_valid_and_test(self): score, breakdown = compute_unified_score(None, None) assert score is None assert breakdown == {} def test_split_quality_rewards_efficiency(self): efficient = MetricsBundle( trade_count=30, profit_factor=2.0, total_return_pct=1.2, max_drawdown_pct=0.4, sharpe_ratio=2.5, win_rate=0.60, monthly_win_rate=0.75, equity_curve_r_squared=0.70, avg_gross_exposure_pct=3.0, avg_net_exposure_pct=-1.0, days_in_market_pct=55.0, ) inefficient = efficient.model_copy( update={"avg_gross_exposure_pct": 8.0, "days_in_market_pct": 85.0} ) efficient_score, _ = compute_unified_split_quality(efficient) inefficient_score, _ = compute_unified_split_quality(inefficient) assert efficient_score is not None assert inefficient_score is not None assert efficient_score > inefficient_score def test_overfit_strategy_scores_below_robust_strategy(self): overfit_test = SplitResult( run_id="bt_step35_test", trade_count=56, profit_factor=2.009, total_return_pct=2.181, win_rate=0.607, max_drawdown_pct=0.442, sharpe_ratio=4.217, monthly_win_rate=1.0, equity_curve_r_squared=0.889, avg_gross_exposure_pct=6.228, avg_net_exposure_pct=1.126, days_in_market_pct=76.6, ) overfit_valid = SplitResult( run_id="bt_step35_valid", trade_count=53, profit_factor=1.243, total_return_pct=0.708, win_rate=0.509, max_drawdown_pct=0.885, sharpe_ratio=1.180, monthly_win_rate=0.75, equity_curve_r_squared=0.146, avg_gross_exposure_pct=7.511, avg_net_exposure_pct=1.567, days_in_market_pct=77.2, ) robust_test = SplitResult( run_id="bt_step52_test", trade_count=22, profit_factor=3.352, total_return_pct=1.011, win_rate=0.773, max_drawdown_pct=0.243, sharpe_ratio=3.032, monthly_win_rate=0.667, equity_curve_r_squared=0.847, avg_gross_exposure_pct=2.858, avg_net_exposure_pct=-2.223, days_in_market_pct=53.2, ) robust_valid = SplitResult( run_id="bt_step52_valid", trade_count=25, profit_factor=5.660, total_return_pct=1.819, win_rate=0.72, max_drawdown_pct=0.217, sharpe_ratio=4.756, monthly_win_rate=1.0, equity_curve_r_squared=0.661, avg_gross_exposure_pct=3.584, avg_net_exposure_pct=-2.593, days_in_market_pct=56.1, ) overfit_score, overfit_breakdown = compute_unified_score(overfit_test, overfit_valid) robust_score, robust_breakdown = compute_unified_score(robust_test, robust_valid) assert overfit_score is not None assert robust_score is not None assert robust_score > overfit_score assert overfit_breakdown["gap_quality"] < robust_breakdown["gap_quality"] class TestComputePublicSQS: def test_requires_walk_forward_and_robustness(self): # v9 returns the first missing requirement (sequential, not all at once) train_result = SplitResult(run_id="bt_train", trade_count=10, total_return_pct=20.0) valid_result = SplitResult(run_id="bt_valid", trade_count=8, total_return_pct=10.0) test_result = SplitResult(run_id="bt_test", trade_count=9, total_return_pct=12.0) public_score, breakdown, source = compute_public_sqs( train_result, valid_result, test_result, deployment_score=55.0, rqs_score=70.0, ) assert public_score is None assert source == "pending_validation" # v9: sequential checks — first missing requirement is walk-forward assert breakdown["requires_walk_forward"] == 1.0 def test_v3_removes_stress_quality_penalty_from_primary_rank(self): train = SplitResult(run_id="bt_train", trade_count=20, total_return_pct=20.0, win_rate=0.7) valid = SplitResult(run_id="bt_valid", trade_count=15, total_return_pct=10.0, win_rate=0.6) test = SplitResult( run_id="bt_test", trade_count=24, total_return_pct=12.0, win_rate=0.6, days_in_market_pct=55.0, ) summary = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=8, test_aggregate=WalkForwardAggregate( mean_return_pct=10.0, median_return_pct=8.0, worst_return_pct=1.0, positive_fold_rate_pct=87.5, mean_profit_factor=1.8, mean_max_drawdown_pct=5.0, mean_trade_count=12.0, mean_win_rate=0.55, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=20.0, fold_return_cv=0.3, ), ) robustness = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=90.0, overall_worst_return_pct=-2.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, mean_return_pct=4.0, median_return_pct=3.5, worst_return_pct=-1.0, positive_window_rate_pct=83.3, mean_max_drawdown_pct=3.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=18.0, median_return_pct=12.0, worst_return_pct=4.0, positive_window_rate_pct=100.0, mean_max_drawdown_pct=5.0, ), ], ) weak_stress = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=55.0, overall_worst_return_pct=-12.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, mean_return_pct=2.0, median_return_pct=1.0, worst_return_pct=-5.0, positive_window_rate_pct=54.5, mean_max_drawdown_pct=4.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=8.0, median_return_pct=5.0, worst_return_pct=-12.0, positive_window_rate_pct=61.5, mean_max_drawdown_pct=10.0, ), ], ) legacy_score, legacy_breakdown, _ = compute_public_sqs_v2( train, valid, test, walk_forward_summary=summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=weak_stress, ) v3_score, v3_breakdown, source = compute_public_sqs_v3( train, valid, test, walk_forward_summary=summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=weak_stress, ) assert legacy_score is not None assert v3_score is not None assert v3_score > legacy_score assert source == "v3_deployment+robustness+oot_gate" assert "oot_quality" in v3_breakdown assert "oot_quality_factor" not in v3_breakdown def test_requires_both_validations_even_if_robustness_exists(self): train_result = SplitResult(run_id="bt_train", trade_count=10, total_return_pct=20.0) valid_result = SplitResult(run_id="bt_valid", trade_count=8, total_return_pct=10.0) test_result = SplitResult(run_id="bt_test", trade_count=9, total_return_pct=12.0) robustness = RobustnessMatrixSummary( horizons_days=[21, 63, 252], step_days=21, overall_window_count=12, overall_positive_window_rate_pct=70.0, overall_worst_return_pct=-15.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=4, mean_return_pct=5.0, median_return_pct=4.0, worst_return_pct=-2.0, positive_window_rate_pct=75.0, mean_max_drawdown_pct=3.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=3, mean_return_pct=12.0, median_return_pct=9.0, worst_return_pct=4.0, positive_window_rate_pct=100.0, mean_max_drawdown_pct=5.0, ), ], ) public_score, breakdown, source = compute_public_sqs( train_result, valid_result, test_result, deployment_score=50.0, robustness_matrix_summary=robustness, rqs_score=70.0, ) assert public_score is None assert source == "pending_validation" # v9: sequential checks — first missing requirement is walk-forward assert breakdown["requires_walk_forward"] == 1.0 def test_v4_falls_back_to_v3_without_common_window(self): train = SplitResult(run_id="bt_train", trade_count=20, total_return_pct=20.0, win_rate=0.7) valid = SplitResult(run_id="bt_valid", trade_count=15, total_return_pct=10.0, win_rate=0.6) test = SplitResult( run_id="bt_test", trade_count=24, total_return_pct=12.0, win_rate=0.6, days_in_market_pct=55.0, ) summary = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=8, test_aggregate=WalkForwardAggregate( mean_return_pct=10.0, median_return_pct=8.0, worst_return_pct=1.0, positive_fold_rate_pct=87.5, mean_profit_factor=1.8, mean_max_drawdown_pct=5.0, mean_trade_count=12.0, mean_win_rate=0.55, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=20.0, fold_return_cv=0.3, ), ) robustness = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=90.0, overall_worst_return_pct=-2.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, mean_return_pct=4.0, median_return_pct=3.5, worst_return_pct=-1.0, positive_window_rate_pct=83.3, mean_max_drawdown_pct=3.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=18.0, median_return_pct=12.0, worst_return_pct=4.0, positive_window_rate_pct=100.0, mean_max_drawdown_pct=5.0, ), ], ) oot = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=70.0, overall_worst_return_pct=-9.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, mean_return_pct=2.0, median_return_pct=1.5, worst_return_pct=-4.0, positive_window_rate_pct=60.0, mean_max_drawdown_pct=4.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=7.0, median_return_pct=4.5, worst_return_pct=-9.0, positive_window_rate_pct=75.0, mean_max_drawdown_pct=8.0, ), ], ) v3_score, _, _ = compute_public_sqs_v3( train, valid, test, walk_forward_summary=summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=oot, ) v4_score, _, source = compute_public_sqs_v4( train, valid, test, walk_forward_summary=summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=oot, common_window_summary=None, ) assert v3_score == v4_score assert source == "v4_fallback_v3_missing_common_window" class TestCommonWindowScore: def test_rewards_higher_return_and_capital_efficiency(self): compounder = CommonWindowSummary( snapshot_id="snap_a", start_date=dt.date(2022, 3, 3), end_date=dt.date(2026, 3, 13), metrics=MetricsBundle( trade_count=180, profit_factor=8.2, total_return_pct=262.9, max_drawdown_pct=4.46, sharpe_ratio=2.68, avg_gross_exposure_pct=25.9, days_in_market_pct=67.1, ), ) slower = CommonWindowSummary( snapshot_id="snap_b", start_date=dt.date(2022, 3, 3), end_date=dt.date(2026, 3, 13), metrics=MetricsBundle( trade_count=146, profit_factor=8.7, total_return_pct=130.1, max_drawdown_pct=5.38, sharpe_ratio=2.38, avg_gross_exposure_pct=36.5, days_in_market_pct=73.7, ), ) compounder_score, compounder_breakdown = compute_common_window_score(compounder) slower_score, _ = compute_common_window_score(slower) assert compounder_score is not None assert slower_score is not None assert compounder_score > slower_score assert compounder_breakdown["cw_return_on_gross"] > 90.0 def test_reset_common_window_rewards_consistent_segments(self): consistent = ResetCommonWindowSummary( snapshot_id="snap_consistent", start_date=dt.date(2022, 3, 3), end_date=dt.date(2026, 3, 13), segment_days=252, segment_summaries=[ CommonWindowSummary( snapshot_id=f"seg_{idx}", start_date=dt.date(2022 + idx, 3, 3), end_date=dt.date(2022 + idx, 12, 31), metrics=MetricsBundle( trade_count=40, profit_factor=2.0, total_return_pct=24.0 + idx, max_drawdown_pct=4.0, sharpe_ratio=1.8, avg_gross_exposure_pct=28.0, days_in_market_pct=55.0, ), ) for idx in range(4) ], ) lumpy = ResetCommonWindowSummary( snapshot_id="snap_lumpy", start_date=dt.date(2022, 3, 3), end_date=dt.date(2026, 3, 13), segment_days=252, segment_summaries=[ CommonWindowSummary( snapshot_id="seg_a", start_date=dt.date(2022, 3, 3), end_date=dt.date(2022, 12, 31), metrics=MetricsBundle( trade_count=40, profit_factor=2.6, total_return_pct=78.0, max_drawdown_pct=4.2, sharpe_ratio=2.1, avg_gross_exposure_pct=28.0, days_in_market_pct=55.0, ), ), CommonWindowSummary( snapshot_id="seg_b", start_date=dt.date(2023, 1, 1), end_date=dt.date(2023, 12, 31), metrics=MetricsBundle( trade_count=40, profit_factor=1.4, total_return_pct=5.0, max_drawdown_pct=4.0, sharpe_ratio=1.1, avg_gross_exposure_pct=28.0, days_in_market_pct=55.0, ), ), CommonWindowSummary( snapshot_id="seg_c", start_date=dt.date(2024, 1, 1), end_date=dt.date(2024, 12, 31), metrics=MetricsBundle( trade_count=40, profit_factor=1.3, total_return_pct=4.0, max_drawdown_pct=4.0, sharpe_ratio=1.0, avg_gross_exposure_pct=28.0, days_in_market_pct=55.0, ), ), CommonWindowSummary( snapshot_id="seg_d", start_date=dt.date(2025, 1, 1), end_date=dt.date(2025, 12, 31), metrics=MetricsBundle( trade_count=40, profit_factor=1.2, total_return_pct=3.0, max_drawdown_pct=4.0, sharpe_ratio=0.9, avg_gross_exposure_pct=28.0, days_in_market_pct=55.0, ), ), ], ) consistent_score, consistent_breakdown = compute_reset_common_window_score(consistent) lumpy_score, _ = compute_reset_common_window_score(lumpy) assert consistent_score is not None assert lumpy_score is not None assert consistent_score > lumpy_score assert consistent_breakdown["rcw_segment_count"] == 4.0 def test_v4_can_promote_compounder_when_v3_gap_is_small(self): train = SplitResult(run_id="bt_train", trade_count=100, total_return_pct=50.0, win_rate=0.7) valid = SplitResult(run_id="bt_valid", trade_count=25, total_return_pct=40.0, win_rate=0.7) test = SplitResult( run_id="bt_test", trade_count=24, total_return_pct=60.0, win_rate=0.7, days_in_market_pct=75.0, ) summary = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=8, test_aggregate=WalkForwardAggregate( mean_return_pct=12.0, median_return_pct=9.0, worst_return_pct=1.0, positive_fold_rate_pct=87.5, mean_profit_factor=2.0, mean_max_drawdown_pct=4.5, mean_trade_count=14.0, mean_win_rate=0.58, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=20.0, fold_return_cv=0.3, ), ) robustness = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=12, overall_positive_window_rate_pct=85.0, overall_worst_return_pct=-3.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=8, mean_return_pct=4.0, median_return_pct=4.0, worst_return_pct=-2.0, positive_window_rate_pct=80.0, mean_max_drawdown_pct=3.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=15.0, median_return_pct=10.0, worst_return_pct=2.0, positive_window_rate_pct=100.0, mean_max_drawdown_pct=5.0, ), ], ) oot = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=65.0, overall_worst_return_pct=-10.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, mean_return_pct=1.5, median_return_pct=1.0, worst_return_pct=-4.0, positive_window_rate_pct=60.0, mean_max_drawdown_pct=4.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=6.0, median_return_pct=4.0, worst_return_pct=-10.0, positive_window_rate_pct=75.0, mean_max_drawdown_pct=8.0, ), ], ) weak_common = CommonWindowSummary( snapshot_id="snap_a", start_date=dt.date(2022, 3, 3), end_date=dt.date(2026, 3, 13), metrics=MetricsBundle( trade_count=140, profit_factor=7.5, total_return_pct=130.1, max_drawdown_pct=5.38, sharpe_ratio=2.38, avg_gross_exposure_pct=36.5, days_in_market_pct=73.7, ), ) strong_common = CommonWindowSummary( snapshot_id="snap_b", start_date=dt.date(2022, 3, 3), end_date=dt.date(2026, 3, 13), metrics=MetricsBundle( trade_count=180, profit_factor=8.2, total_return_pct=262.9, max_drawdown_pct=4.46, sharpe_ratio=2.68, avg_gross_exposure_pct=25.9, days_in_market_pct=67.1, ), ) v4_weak, _, _ = compute_public_sqs_v4( train, valid, test, walk_forward_summary=summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=oot, common_window_summary=weak_common, ) v4_strong, _, _ = compute_public_sqs_v4( train, valid, test, walk_forward_summary=summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=oot, common_window_summary=strong_common, ) assert v4_weak is not None assert v4_strong is not None assert v4_strong > v4_weak def test_v6_prefers_reset_common_window_when_available(self): train = SplitResult(run_id="bt_train", trade_count=100, total_return_pct=50.0, win_rate=0.7) valid = SplitResult(run_id="bt_valid", trade_count=25, total_return_pct=40.0, win_rate=0.7) test = SplitResult( run_id="bt_test", trade_count=24, total_return_pct=60.0, win_rate=0.7, days_in_market_pct=75.0, ) summary = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=8, test_aggregate=WalkForwardAggregate( mean_return_pct=12.0, median_return_pct=9.0, worst_return_pct=1.0, positive_fold_rate_pct=87.5, mean_profit_factor=2.0, mean_max_drawdown_pct=4.5, mean_trade_count=14.0, mean_win_rate=0.58, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=20.0, fold_return_cv=0.3, ), ) robustness = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=12, overall_positive_window_rate_pct=85.0, overall_worst_return_pct=-3.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=8, mean_return_pct=4.0, median_return_pct=4.0, worst_return_pct=-2.0, positive_window_rate_pct=80.0, mean_max_drawdown_pct=3.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=15.0, median_return_pct=10.0, worst_return_pct=2.0, positive_window_rate_pct=100.0, mean_max_drawdown_pct=5.0, ), ], ) oot = robustness.model_copy() common = CommonWindowSummary( snapshot_id="snap10k", start_date=dt.date(2022, 3, 3), end_date=dt.date(2026, 3, 13), initial_equity=10_000.0, metrics=MetricsBundle( trade_count=180, profit_factor=8.2, total_return_pct=262.9, max_drawdown_pct=4.46, sharpe_ratio=2.68, avg_gross_exposure_pct=25.9, days_in_market_pct=67.1, ), ) weak_reset = ResetCommonWindowSummary( snapshot_id="reset_weak", start_date=dt.date(2022, 3, 3), end_date=dt.date(2026, 3, 13), segment_days=252, segment_summaries=[ common.model_copy(update={"snapshot_id": "seg0", "metrics": common.metrics.model_copy(update={"total_return_pct": 10.0, "profit_factor": 1.2, "sharpe_ratio": 0.8})}), common.model_copy(update={"snapshot_id": "seg1", "metrics": common.metrics.model_copy(update={"total_return_pct": 8.0, "profit_factor": 1.1, "sharpe_ratio": 0.7})}), common.model_copy(update={"snapshot_id": "seg2", "metrics": common.metrics.model_copy(update={"total_return_pct": 6.0, "profit_factor": 1.1, "sharpe_ratio": 0.7})}), common.model_copy(update={"snapshot_id": "seg3", "metrics": common.metrics.model_copy(update={"total_return_pct": 4.0, "profit_factor": 1.0, "sharpe_ratio": 0.6})}), ], ) strong_reset = ResetCommonWindowSummary( snapshot_id="reset_strong", start_date=dt.date(2022, 3, 3), end_date=dt.date(2026, 3, 13), segment_days=252, segment_summaries=[ common.model_copy(update={"snapshot_id": "seg0", "metrics": common.metrics.model_copy(update={"total_return_pct": 24.0, "profit_factor": 2.0, "sharpe_ratio": 1.8})}), common.model_copy(update={"snapshot_id": "seg1", "metrics": common.metrics.model_copy(update={"total_return_pct": 25.0, "profit_factor": 2.1, "sharpe_ratio": 1.9})}), common.model_copy(update={"snapshot_id": "seg2", "metrics": common.metrics.model_copy(update={"total_return_pct": 23.0, "profit_factor": 2.0, "sharpe_ratio": 1.8})}), common.model_copy(update={"snapshot_id": "seg3", "metrics": common.metrics.model_copy(update={"total_return_pct": 24.0, "profit_factor": 2.1, "sharpe_ratio": 1.9})}), ], ) weak_v6, _, weak_source = compute_public_sqs_v6( train, valid, test, walk_forward_summary=summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=oot, common_window_summary=common, reset_common_window_summary=weak_reset, ) strong_v6, _, strong_source = compute_public_sqs( train, valid, test, walk_forward_summary=summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=oot, common_window_summary=common, reset_common_window_summary=strong_reset, ) assert weak_v6 is not None assert strong_v6 is not None assert weak_source == "v6_deployment+reset_common_window" # v6 directly: unchanged assert strong_source == "v9_3pillar+reset_cw[oot_quality_fallback]" # default now v9 assert strong_v6 > weak_v6 def test_v6_falls_back_to_v4_for_noncomparable_reset_common_window(self): train = SplitResult(run_id="bt_train", trade_count=100, total_return_pct=50.0, win_rate=0.7) valid = SplitResult(run_id="bt_valid", trade_count=25, total_return_pct=40.0, win_rate=0.7) test = SplitResult( run_id="bt_test", trade_count=24, total_return_pct=60.0, win_rate=0.7, days_in_market_pct=75.0, ) summary = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=8, test_aggregate=WalkForwardAggregate( mean_return_pct=12.0, median_return_pct=9.0, worst_return_pct=1.0, positive_fold_rate_pct=87.5, mean_profit_factor=2.0, mean_max_drawdown_pct=4.5, mean_trade_count=14.0, mean_win_rate=0.58, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=20.0, fold_return_cv=0.3, ), ) robustness = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=12, overall_positive_window_rate_pct=85.0, overall_worst_return_pct=-3.0, horizon_summaries=[], ) common = CommonWindowSummary( snapshot_id="snap10k", start_date=dt.date(2022, 3, 3), end_date=dt.date(2026, 3, 13), initial_equity=10_000.0, metrics=MetricsBundle( trade_count=180, profit_factor=8.2, total_return_pct=262.9, max_drawdown_pct=4.46, sharpe_ratio=2.68, avg_gross_exposure_pct=25.9, days_in_market_pct=67.1, ), ) noncomparable_reset = ResetCommonWindowSummary( snapshot_id="reset_bad", start_date=dt.date(2022, 3, 3), end_date=dt.date(2026, 3, 13), reset_initial_equity=25_000.0, segment_days=252, segment_summaries=[common.model_copy(update={"initial_equity": 25_000.0})], ) default_score, _, default_source = compute_public_sqs( train, valid, test, walk_forward_summary=summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=robustness, common_window_summary=common, reset_common_window_summary=noncomparable_reset, ) v4_score, _, v4_source = compute_public_sqs_v4( train, valid, test, walk_forward_summary=summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=robustness, common_window_summary=common, ) # v9 falls back to common_window blend when reset is noncomparable assert default_score is not None assert default_source == "v9_3pillar+cw[oot_quality_fallback]" # v4 still returns the old source when called directly assert v4_source == "v4_deployment+common_window" def test_v4_treats_sparse_zero_trade_oot_as_neutral(self): train = SplitResult(run_id="bt_train", trade_count=100, total_return_pct=50.0, win_rate=0.7) valid = SplitResult(run_id="bt_valid", trade_count=25, total_return_pct=40.0, win_rate=0.7) test = SplitResult( run_id="bt_test", trade_count=24, total_return_pct=60.0, win_rate=0.7, days_in_market_pct=75.0, ) summary = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=8, test_aggregate=WalkForwardAggregate( mean_return_pct=12.0, median_return_pct=9.0, worst_return_pct=1.0, positive_fold_rate_pct=87.5, mean_profit_factor=2.0, mean_max_drawdown_pct=4.5, mean_trade_count=14.0, mean_win_rate=0.58, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=20.0, fold_return_cv=0.3, ), ) robustness = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=12, overall_positive_window_rate_pct=85.0, overall_worst_return_pct=-3.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=8, mean_return_pct=4.0, median_return_pct=4.0, worst_return_pct=-2.0, positive_window_rate_pct=80.0, mean_max_drawdown_pct=3.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=15.0, median_return_pct=10.0, worst_return_pct=2.0, positive_window_rate_pct=100.0, mean_max_drawdown_pct=5.0, ), ], ) sparse_oot = RobustnessMatrixSummary( horizons_days=[21, 63, 252], step_days=21, overall_window_count=12, overall_positive_window_rate_pct=0.0, overall_worst_return_pct=0.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=21, window_count=5, mean_return_pct=0.0, median_return_pct=0.0, worst_return_pct=0.0, positive_window_rate_pct=0.0, mean_max_drawdown_pct=0.0, ), RobustnessHorizonSummary( horizon_days=63, window_count=4, mean_return_pct=0.0, median_return_pct=0.0, worst_return_pct=0.0, positive_window_rate_pct=0.0, mean_max_drawdown_pct=0.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=3, mean_return_pct=0.0, median_return_pct=0.0, worst_return_pct=0.0, positive_window_rate_pct=0.0, mean_max_drawdown_pct=0.0, ), ], ) common = CommonWindowSummary( snapshot_id="snap10k", start_date=dt.date(2022, 3, 3), end_date=dt.date(2026, 3, 24), initial_equity=10_000.0, metrics=MetricsBundle( trade_count=146, profit_factor=12.0, total_return_pct=508.7, max_drawdown_pct=5.59, sharpe_ratio=3.25, avg_gross_exposure_pct=30.0, days_in_market_pct=60.0, ), ) score, breakdown, source = compute_public_sqs_v4( train, valid, test, walk_forward_summary=summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=sparse_oot, common_window_summary=common, ) assert source == "v4_deployment+common_window" assert score is not None assert score > 45.0 assert breakdown["oot_gate_factor"] == 1.0 def test_v4_falls_back_to_v3_for_noncomparable_common_window_equity(self): train = SplitResult(run_id="bt_train", trade_count=100, total_return_pct=50.0, win_rate=0.7) valid = SplitResult(run_id="bt_valid", trade_count=25, total_return_pct=40.0, win_rate=0.7) test = SplitResult( run_id="bt_test", trade_count=24, total_return_pct=60.0, win_rate=0.7, days_in_market_pct=75.0, ) summary = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=8, test_aggregate=WalkForwardAggregate( mean_return_pct=12.0, median_return_pct=9.0, worst_return_pct=1.0, positive_fold_rate_pct=87.5, mean_profit_factor=2.0, mean_max_drawdown_pct=4.5, mean_trade_count=14.0, mean_win_rate=0.58, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=20.0, fold_return_cv=0.3, ), ) robustness = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=12, overall_positive_window_rate_pct=85.0, overall_worst_return_pct=-3.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=8, mean_return_pct=4.0, median_return_pct=4.0, worst_return_pct=-2.0, positive_window_rate_pct=80.0, mean_max_drawdown_pct=3.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=15.0, median_return_pct=10.0, worst_return_pct=2.0, positive_window_rate_pct=100.0, mean_max_drawdown_pct=5.0, ), ], ) oot = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=65.0, overall_worst_return_pct=-10.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, mean_return_pct=1.5, median_return_pct=1.0, worst_return_pct=-4.0, positive_window_rate_pct=60.0, mean_max_drawdown_pct=4.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=6.0, median_return_pct=4.0, worst_return_pct=-10.0, positive_window_rate_pct=75.0, mean_max_drawdown_pct=8.0, ), ], ) noncomparable_common = CommonWindowSummary( snapshot_id="snap_old_capital", start_date=dt.date(2022, 3, 3), end_date=dt.date(2026, 3, 13), initial_equity=100_000.0, metrics=MetricsBundle( trade_count=180, profit_factor=8.2, total_return_pct=262.9, max_drawdown_pct=4.46, sharpe_ratio=2.68, avg_gross_exposure_pct=25.9, days_in_market_pct=67.1, ), ) v3, _, _ = compute_public_sqs_v3( train, valid, test, walk_forward_summary=summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=oot, ) v4, _, source = compute_public_sqs_v4( train, valid, test, walk_forward_summary=summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=oot, common_window_summary=noncomparable_common, ) assert v4 == v3 assert source == "v4_fallback_v3_noncomparable_common_window" def test_multi_capital_common_window_blends_10k_25k_100k_with_floor(self): cw_10k = CommonWindowSummary( snapshot_id="snap_10k", start_date=dt.date(2022, 3, 3), end_date=dt.date(2026, 3, 13), initial_equity=10_000.0, metrics=MetricsBundle( trade_count=180, profit_factor=8.2, total_return_pct=262.9, max_drawdown_pct=4.46, sharpe_ratio=2.68, avg_gross_exposure_pct=25.9, days_in_market_pct=67.1, ), ) cw_25k = CommonWindowSummary( snapshot_id="snap_25k", start_date=dt.date(2022, 3, 3), end_date=dt.date(2026, 3, 13), initial_equity=25_000.0, metrics=MetricsBundle( trade_count=176, profit_factor=7.8, total_return_pct=240.0, max_drawdown_pct=4.8, sharpe_ratio=2.55, avg_gross_exposure_pct=28.0, days_in_market_pct=68.5, ), ) cw_100k = CommonWindowSummary( snapshot_id="snap_100k", start_date=dt.date(2022, 3, 3), end_date=dt.date(2026, 3, 13), initial_equity=100_000.0, metrics=MetricsBundle( trade_count=170, profit_factor=7.2, total_return_pct=225.0, max_drawdown_pct=5.1, sharpe_ratio=2.41, avg_gross_exposure_pct=29.8, days_in_market_pct=69.2, ), ) multi = MultiCapitalCommonWindowSummary( capital_summaries=[cw_10k, cw_25k, cw_100k] ) score_10k, _ = compute_common_window_score(cw_10k) score_25k, _ = compute_common_window_score(cw_25k) score_100k, _ = compute_common_window_score(cw_100k) score, breakdown = compute_multi_capital_common_window_score(multi) expected_weighted = ( score_10k * 0.60 + score_25k * 0.25 + score_100k * 0.15 ) expected = round(expected_weighted * 0.90 + min(score_10k, score_25k, score_100k) * 0.10, 1) assert score == expected assert breakdown["mcw_10k_score"] == round(score_10k, 1) assert breakdown["mcw_25k_score"] == round(score_25k, 1) assert breakdown["mcw_100k_score"] == round(score_100k, 1) def test_v5_promotes_better_multi_capital_profile(self): train = SplitResult(run_id="bt_train", trade_count=100, total_return_pct=50.0, win_rate=0.7) valid = SplitResult(run_id="bt_valid", trade_count=25, total_return_pct=40.0, win_rate=0.7) test = SplitResult( run_id="bt_test", trade_count=24, total_return_pct=60.0, win_rate=0.7, days_in_market_pct=75.0, ) summary = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=8, test_aggregate=WalkForwardAggregate( mean_return_pct=12.0, median_return_pct=9.0, worst_return_pct=1.0, positive_fold_rate_pct=87.5, mean_profit_factor=2.0, mean_max_drawdown_pct=4.5, mean_trade_count=14.0, mean_win_rate=0.58, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=20.0, fold_return_cv=0.3, ), ) robustness = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=12, overall_positive_window_rate_pct=85.0, overall_worst_return_pct=-3.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=8, mean_return_pct=4.0, median_return_pct=4.0, worst_return_pct=-2.0, positive_window_rate_pct=80.0, mean_max_drawdown_pct=3.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=15.0, median_return_pct=10.0, worst_return_pct=2.0, positive_window_rate_pct=100.0, mean_max_drawdown_pct=5.0, ), ], ) oot = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=65.0, overall_worst_return_pct=-10.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, mean_return_pct=1.5, median_return_pct=1.0, worst_return_pct=-4.0, positive_window_rate_pct=60.0, mean_max_drawdown_pct=4.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=6.0, median_return_pct=4.0, worst_return_pct=-10.0, positive_window_rate_pct=75.0, mean_max_drawdown_pct=8.0, ), ], ) weak_10k = CommonWindowSummary( snapshot_id="weak10k", start_date=dt.date(2022, 3, 3), end_date=dt.date(2026, 3, 13), initial_equity=10_000.0, metrics=MetricsBundle( trade_count=140, profit_factor=6.0, total_return_pct=120.0, max_drawdown_pct=6.2, sharpe_ratio=2.0, avg_gross_exposure_pct=34.0, days_in_market_pct=73.7, ), ) weak_25k = weak_10k.model_copy(update={"snapshot_id": "weak25k", "initial_equity": 25_000.0}) weak_100k = weak_10k.model_copy(update={"snapshot_id": "weak100k", "initial_equity": 100_000.0}) strong_10k = CommonWindowSummary( snapshot_id="strong10k", start_date=dt.date(2022, 3, 3), end_date=dt.date(2026, 3, 13), initial_equity=10_000.0, metrics=MetricsBundle( trade_count=180, profit_factor=8.2, total_return_pct=262.9, max_drawdown_pct=4.46, sharpe_ratio=2.68, avg_gross_exposure_pct=25.9, days_in_market_pct=67.1, ), ) strong_25k = strong_10k.model_copy(update={"snapshot_id": "strong25k", "initial_equity": 25_000.0}) strong_100k = strong_10k.model_copy(update={"snapshot_id": "strong100k", "initial_equity": 100_000.0}) weak_multi = MultiCapitalCommonWindowSummary( capital_summaries=[weak_10k, weak_25k, weak_100k] ) strong_multi = MultiCapitalCommonWindowSummary( capital_summaries=[strong_10k, strong_25k, strong_100k] ) weak_v5, _, source = compute_public_sqs_v5( train, valid, test, walk_forward_summary=summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=oot, common_window_summary=strong_10k, multi_capital_common_window_summary=weak_multi, ) strong_v5, _, strong_source = compute_public_sqs_v5( train, valid, test, walk_forward_summary=summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=oot, common_window_summary=strong_10k, multi_capital_common_window_summary=strong_multi, ) assert source == "v5_deployment+multi_capital_common_window" assert strong_source == "v5_deployment+multi_capital_common_window" assert strong_v5 is not None assert weak_v5 is not None assert strong_v5 > weak_v5 def test_v5_falls_back_to_v4_without_comparable_multi_capital_runs(self): train = SplitResult(run_id="bt_train", trade_count=100, total_return_pct=50.0, win_rate=0.7) valid = SplitResult(run_id="bt_valid", trade_count=25, total_return_pct=40.0, win_rate=0.7) test = SplitResult( run_id="bt_test", trade_count=24, total_return_pct=60.0, win_rate=0.7, days_in_market_pct=75.0, ) summary = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=8, test_aggregate=WalkForwardAggregate( mean_return_pct=12.0, median_return_pct=9.0, worst_return_pct=1.0, positive_fold_rate_pct=87.5, mean_profit_factor=2.0, mean_max_drawdown_pct=4.5, mean_trade_count=14.0, mean_win_rate=0.58, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=20.0, fold_return_cv=0.3, ), ) robustness = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=12, overall_positive_window_rate_pct=85.0, overall_worst_return_pct=-3.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=8, mean_return_pct=4.0, median_return_pct=4.0, worst_return_pct=-2.0, positive_window_rate_pct=80.0, mean_max_drawdown_pct=3.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=15.0, median_return_pct=10.0, worst_return_pct=2.0, positive_window_rate_pct=100.0, mean_max_drawdown_pct=5.0, ), ], ) oot = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=65.0, overall_worst_return_pct=-10.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, mean_return_pct=1.5, median_return_pct=1.0, worst_return_pct=-4.0, positive_window_rate_pct=60.0, mean_max_drawdown_pct=4.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=6.0, median_return_pct=4.0, worst_return_pct=-10.0, positive_window_rate_pct=75.0, mean_max_drawdown_pct=8.0, ), ], ) comparable_10k = CommonWindowSummary( snapshot_id="snap10k", start_date=dt.date(2022, 3, 3), end_date=dt.date(2026, 3, 13), initial_equity=10_000.0, metrics=MetricsBundle( trade_count=180, profit_factor=8.2, total_return_pct=262.9, max_drawdown_pct=4.46, sharpe_ratio=2.68, avg_gross_exposure_pct=25.9, days_in_market_pct=67.1, ), ) incomplete_multi = MultiCapitalCommonWindowSummary( capital_summaries=[ comparable_10k, comparable_10k.model_copy(update={"snapshot_id": "snap25k", "initial_equity": 25_000.0}), ] ) v4, _, _ = compute_public_sqs_v4( train, valid, test, walk_forward_summary=summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=oot, common_window_summary=comparable_10k, ) v5, _, source = compute_public_sqs_v5( train, valid, test, walk_forward_summary=summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=oot, common_window_summary=comparable_10k, multi_capital_common_window_summary=incomplete_multi, ) assert v5 == v4 assert source == "v4_deployment+common_window" def test_default_public_sqs_uses_v4_even_when_multi_capital_summary_exists(self): train = SplitResult(run_id="bt_train", trade_count=100, total_return_pct=50.0, win_rate=0.7) valid = SplitResult(run_id="bt_valid", trade_count=25, total_return_pct=40.0, win_rate=0.7) test = SplitResult( run_id="bt_test", trade_count=24, total_return_pct=60.0, win_rate=0.7, days_in_market_pct=75.0, ) summary = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=8, test_aggregate=WalkForwardAggregate( mean_return_pct=12.0, median_return_pct=9.0, worst_return_pct=1.0, positive_fold_rate_pct=87.5, mean_profit_factor=2.0, mean_max_drawdown_pct=4.5, mean_trade_count=14.0, mean_win_rate=0.58, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=20.0, fold_return_cv=0.3, ), ) robustness = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=12, overall_positive_window_rate_pct=85.0, overall_worst_return_pct=-3.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=8, mean_return_pct=4.0, median_return_pct=4.0, worst_return_pct=-2.0, positive_window_rate_pct=80.0, mean_max_drawdown_pct=3.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=15.0, median_return_pct=10.0, worst_return_pct=2.0, positive_window_rate_pct=100.0, mean_max_drawdown_pct=5.0, ), ], ) oot = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=65.0, overall_worst_return_pct=-10.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, mean_return_pct=1.5, median_return_pct=1.0, worst_return_pct=-4.0, positive_window_rate_pct=60.0, mean_max_drawdown_pct=4.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=6.0, median_return_pct=4.0, worst_return_pct=-10.0, positive_window_rate_pct=75.0, mean_max_drawdown_pct=8.0, ), ], ) comparable_10k = CommonWindowSummary( snapshot_id="snap10k", start_date=dt.date(2022, 3, 3), end_date=dt.date(2026, 3, 13), initial_equity=10_000.0, metrics=MetricsBundle( trade_count=180, profit_factor=8.2, total_return_pct=262.9, max_drawdown_pct=4.46, sharpe_ratio=2.68, avg_gross_exposure_pct=25.9, days_in_market_pct=67.1, ), ) multi = MultiCapitalCommonWindowSummary( capital_summaries=[ comparable_10k, comparable_10k.model_copy(update={"snapshot_id": "snap25k", "initial_equity": 25_000.0}), comparable_10k.model_copy(update={"snapshot_id": "snap100k", "initial_equity": 100_000.0}), ] ) v4, _, v4_source = compute_public_sqs_v4( train, valid, test, walk_forward_summary=summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=oot, common_window_summary=comparable_10k, ) default_score, _, default_source = compute_public_sqs( train, valid, test, walk_forward_summary=summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=oot, common_window_summary=comparable_10k, multi_capital_common_window_summary=multi, ) # default (v9) uses common_window blend, not multi-capital assert default_score is not None assert default_source == "v9_3pillar+cw[oot_quality_fallback]" # v4 still returns the old source when called directly assert v4_source == "v4_deployment+common_window" class TestOotFactorV2: """Tests for the continuous OOT factor used in SQS v7/v8.""" def _make_oot_summary( self, h63_median: float = 2.0, h252_median: float = 5.0, positive_rate: float = 60.0, worst_return: float = -8.0, ) -> RobustnessMatrixSummary: return RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=20, overall_positive_window_rate_pct=positive_rate, overall_worst_return_pct=worst_return, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=10, mean_return_pct=h63_median, median_return_pct=h63_median, worst_return_pct=worst_return, positive_window_rate_pct=positive_rate, mean_max_drawdown_pct=5.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=10, mean_return_pct=h252_median, median_return_pct=h252_median, worst_return_pct=worst_return, positive_window_rate_pct=positive_rate, mean_max_drawdown_pct=5.0, ), ], ) def test_returns_1_when_summary_is_none(self): factor, _ = compute_oot_factor_v2(None) assert factor == 1.0 def test_factor_range_is_between_0_75_and_1_0(self): """Factor must always be in [0.75, 1.00].""" # Good OOT good = self._make_oot_summary(h252_median=10.0, positive_rate=70.0, worst_return=-5.0) factor_good, _ = compute_oot_factor_v2(good) assert 0.75 <= factor_good <= 1.0 # Bad OOT bad = self._make_oot_summary(h63_median=-8.0, h252_median=-15.0, positive_rate=20.0, worst_return=-30.0) factor_bad, _ = compute_oot_factor_v2(bad) assert 0.75 <= factor_bad <= 1.0 def test_better_oot_gives_higher_factor(self): good = self._make_oot_summary(h252_median=8.0, positive_rate=65.0, worst_return=-6.0) bad = self._make_oot_summary(h252_median=-10.0, positive_rate=25.0, worst_return=-25.0) factor_good, _ = compute_oot_factor_v2(good) factor_bad, _ = compute_oot_factor_v2(bad) assert factor_good > factor_bad def test_near_threshold_cases_are_proportional(self): """v7.120 case: 252d median +1.63% should score proportionally, not cliff to 0.85.""" near_threshold = self._make_oot_summary( h63_median=1.5, h252_median=1.63, positive_rate=52.0, worst_return=-10.0 ) factor, breakdown = compute_oot_factor_v2(near_threshold) # Should be significantly above the binary gate floor of 0.85 assert factor > 0.80 assert "oot_quality" in breakdown def test_v7_higher_than_v3_for_imperfect_oot(self): """A strategy with 3/4 OOT criteria should score higher in v7 than v3 (cliff removed).""" import datetime as dt train = SplitResult(run_id="bt_train", trade_count=100, total_return_pct=50.0, win_rate=0.7) valid = SplitResult(run_id="bt_valid", trade_count=25, total_return_pct=40.0, win_rate=0.7) test = SplitResult( run_id="bt_test", trade_count=24, total_return_pct=60.0, win_rate=0.7, days_in_market_pct=75.0, ) summary = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=8, test_aggregate=WalkForwardAggregate( mean_return_pct=12.0, median_return_pct=9.0, worst_return_pct=1.0, positive_fold_rate_pct=87.5, mean_profit_factor=2.0, mean_max_drawdown_pct=4.5, mean_trade_count=14.0, mean_win_rate=0.58, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=20.0, fold_return_cv=0.3, ), ) robustness = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=30, overall_positive_window_rate_pct=75.0, overall_worst_return_pct=-4.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=15, mean_return_pct=4.0, median_return_pct=4.0, worst_return_pct=-4.0, positive_window_rate_pct=75.0, mean_max_drawdown_pct=3.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=15, mean_return_pct=10.0, median_return_pct=10.0, worst_return_pct=-3.0, positive_window_rate_pct=80.0, mean_max_drawdown_pct=4.0, ), ], ) # OOT that passes 3/4 binary criteria (252d median +1.63% < 3.0%) oot_3of4 = self._make_oot_summary( h63_median=1.5, h252_median=1.63, positive_rate=55.0, worst_return=-10.0 ) v3, _, v3_source = compute_public_sqs_v3( train, valid, test, walk_forward_summary=summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=oot_3of4, ) v7, _, v7_source = compute_public_sqs_v7( train, valid, test, walk_forward_summary=summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=oot_3of4, ) assert v3 is not None assert v7 is not None assert v3_source == "v3_deployment+robustness+oot_gate" assert v7_source == "v7_continuous_oot+3crit_deployment" # v7 should be higher: continuous OOT factor + gap no longer double-penalised assert v7 > v3 def test_deployment_gate_not_penalised_by_gap_in_v7(self): """v7 deployment gate ignores WFV gap (already in WFQS _gap_penalty).""" import datetime as dt train = SplitResult(run_id="bt_train", trade_count=100, total_return_pct=50.0, win_rate=0.7) valid = SplitResult(run_id="bt_valid", trade_count=25, total_return_pct=40.0, win_rate=0.7) test = SplitResult(run_id="bt_test", trade_count=24, total_return_pct=60.0, win_rate=0.7, days_in_market_pct=75.0) robustness = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=20, overall_positive_window_rate_pct=70.0, overall_worst_return_pct=-5.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=10, mean_return_pct=4.0, median_return_pct=4.0, worst_return_pct=-5.0, positive_window_rate_pct=70.0, mean_max_drawdown_pct=3.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=10, mean_return_pct=9.0, median_return_pct=9.0, worst_return_pct=-4.0, positive_window_rate_pct=75.0, mean_max_drawdown_pct=4.0, ), ], ) oot = self._make_oot_summary() # v3: high gap (>35%) fails deployment gate → 3/4 → 0.85 high_gap_summary = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=8, test_aggregate=WalkForwardAggregate( mean_return_pct=10.0, median_return_pct=8.0, worst_return_pct=1.0, positive_fold_rate_pct=87.5, mean_profit_factor=2.0, mean_max_drawdown_pct=4.0, mean_trade_count=14.0, mean_win_rate=0.58, ), gap_stats=WalkForwardGapStats(mean_train_test_return_gap_pct=100.0, fold_return_cv=0.3), ) v3, _, _ = compute_public_sqs_v3( train, valid, test, walk_forward_summary=high_gap_summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=oot, ) v7, _, _ = compute_public_sqs_v7( train, valid, test, walk_forward_summary=high_gap_summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=oot, ) # v7 not additionally penalised for gap in deployment gate (already in WFQS) assert v7 is not None assert v7 > v3 class TestActiveResearchWindow: def test_marks_imp_0606_and_later_as_active(self): assert is_active_research_entry("IMP-0606") is True assert is_active_research_entry("IMP-0797") is True def test_marks_pre_imp_0606_as_retired(self): assert is_active_research_entry("IMP-0605") is False def test_unknown_entry_id_defaults_to_active(self): assert is_active_research_entry("custom-id") is True class TestComputeRQS: def test_requires_valid_and_test(self): score, breakdown = compute_rqs(None, None, None) assert score is None assert breakdown == {} def test_rewards_return_max_profile(self): conservative_train = SplitResult( run_id="bt_cons_train", trade_count=30, profit_factor=2.5, total_return_pct=12.0, annualized_return_pct=6.0, win_rate=0.8, max_drawdown_pct=2.0, sharpe_ratio=2.0, monthly_win_rate=0.8, equity_curve_r_squared=0.8, avg_gross_exposure_pct=10.0, avg_net_exposure_pct=10.0, days_in_market_pct=15.0, ) conservative_valid = SplitResult( run_id="bt_cons_valid", trade_count=12, profit_factor=2.0, total_return_pct=10.0, annualized_return_pct=30.0, win_rate=0.75, max_drawdown_pct=3.0, sharpe_ratio=2.2, monthly_win_rate=1.0, equity_curve_r_squared=0.7, avg_gross_exposure_pct=9.0, avg_net_exposure_pct=9.0, days_in_market_pct=18.0, ) conservative_test = SplitResult( run_id="bt_cons_test", trade_count=14, profit_factor=2.1, total_return_pct=18.0, annualized_return_pct=58.0, win_rate=0.8, max_drawdown_pct=3.5, sharpe_ratio=2.0, monthly_win_rate=1.0, equity_curve_r_squared=0.7, avg_gross_exposure_pct=14.0, avg_net_exposure_pct=14.0, days_in_market_pct=20.0, ) return_max_train = SplitResult( run_id="bt_ret_train", trade_count=13, profit_factor=None, total_return_pct=44.1, annualized_return_pct=18.0, win_rate=1.0, max_drawdown_pct=5.4, sharpe_ratio=1.5, monthly_win_rate=1.0, equity_curve_r_squared=0.76, avg_gross_exposure_pct=11.1, avg_net_exposure_pct=11.1, days_in_market_pct=11.8, ) return_max_valid = SplitResult( run_id="bt_ret_valid", trade_count=7, profit_factor=None, total_return_pct=35.3, annualized_return_pct=105.0, win_rate=1.0, max_drawdown_pct=4.1, sharpe_ratio=3.0, monthly_win_rate=1.0, equity_curve_r_squared=0.81, avg_gross_exposure_pct=17.7, avg_net_exposure_pct=17.7, days_in_market_pct=19.4, ) return_max_test = SplitResult( run_id="bt_ret_test", trade_count=15, profit_factor=5.2, total_return_pct=44.7, annualized_return_pct=125.0, win_rate=0.867, max_drawdown_pct=8.3, sharpe_ratio=2.6, monthly_win_rate=1.0, equity_curve_r_squared=0.87, avg_gross_exposure_pct=42.3, avg_net_exposure_pct=42.3, days_in_market_pct=51.1, ) conservative_score, conservative_breakdown = compute_rqs( conservative_train, conservative_valid, conservative_test, ) return_max_score, return_max_breakdown = compute_rqs( return_max_train, return_max_valid, return_max_test, ) assert conservative_score is not None assert return_max_score is not None assert return_max_score > conservative_score assert return_max_breakdown["train_quality"] > 70.0 assert return_max_breakdown["valid_quality"] > 70.0 assert return_max_breakdown["test_quality"] > conservative_breakdown["test_quality"] def test_missing_train_gets_penalized(self): valid = SplitResult( run_id="bt_valid", trade_count=15, profit_factor=2.5, total_return_pct=20.0, win_rate=0.8, max_drawdown_pct=3.0, sharpe_ratio=2.0, monthly_win_rate=1.0, equity_curve_r_squared=0.7, avg_gross_exposure_pct=15.0, avg_net_exposure_pct=15.0, days_in_market_pct=30.0, ) test = SplitResult( run_id="bt_test", trade_count=15, profit_factor=2.5, total_return_pct=25.0, win_rate=0.8, max_drawdown_pct=3.0, sharpe_ratio=2.0, monthly_win_rate=1.0, equity_curve_r_squared=0.7, avg_gross_exposure_pct=20.0, avg_net_exposure_pct=20.0, days_in_market_pct=35.0, ) train = SplitResult( run_id="bt_train", trade_count=15, profit_factor=2.0, total_return_pct=30.0, win_rate=0.7, max_drawdown_pct=4.0, sharpe_ratio=1.5, monthly_win_rate=0.8, equity_curve_r_squared=0.6, avg_gross_exposure_pct=14.0, avg_net_exposure_pct=14.0, days_in_market_pct=25.0, ) full_score, _ = compute_rqs(train, valid, test) penalized_score, _ = compute_rqs(None, valid, test) assert full_score is not None assert penalized_score is not None assert full_score > penalized_score class TestComputeWFQS: def test_requires_summary(self): score, breakdown = compute_wfqs(None) assert score is None assert breakdown == {} def test_rewards_stable_walk_forward_profile(self): robust = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=8, test_aggregate=WalkForwardAggregate( mean_return_pct=14.0, median_return_pct=12.0, worst_return_pct=2.0, positive_fold_rate_pct=87.5, mean_profit_factor=2.2, mean_max_drawdown_pct=4.0, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=18.0, worst_train_test_return_gap_pct=30.0, ), ) overfit = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=8, test_aggregate=WalkForwardAggregate( mean_return_pct=10.0, median_return_pct=4.0, worst_return_pct=-8.0, positive_fold_rate_pct=62.5, mean_profit_factor=1.3, mean_max_drawdown_pct=8.5, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=55.0, worst_train_test_return_gap_pct=95.0, ), ) robust_score, robust_breakdown = compute_wfqs(robust) overfit_score, _ = compute_wfqs(overfit) assert robust_score is not None assert overfit_score is not None assert robust_score > overfit_score assert robust_breakdown["train_test_gap"] > 50.0 class TestComputeDeploymentScore: def test_requires_walk_forward(self): score, breakdown = compute_deployment_score(None, None, None, None) assert score is None assert breakdown == {} def test_gate_penalizes_overfit_profile(self): train = SplitResult(run_id="bt_train", trade_count=10, total_return_pct=40.0, win_rate=1.0) valid = SplitResult(run_id="bt_valid", trade_count=10, total_return_pct=20.0, win_rate=0.8) test = SplitResult(run_id="bt_test", trade_count=10, total_return_pct=25.0, win_rate=0.8) robust_wf = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=8, test_aggregate=WalkForwardAggregate( mean_return_pct=16.0, median_return_pct=12.0, worst_return_pct=1.0, positive_fold_rate_pct=87.5, mean_profit_factor=2.2, mean_max_drawdown_pct=4.0, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=20.0, worst_train_test_return_gap_pct=35.0, ), ) overfit_wf = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=8, test_aggregate=WalkForwardAggregate( mean_return_pct=16.0, median_return_pct=2.0, worst_return_pct=-7.0, positive_fold_rate_pct=50.0, mean_profit_factor=1.5, mean_max_drawdown_pct=7.0, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=60.0, worst_train_test_return_gap_pct=90.0, ), ) robust_score, robust_breakdown = compute_deployment_score(train, valid, test, robust_wf, rqs_score=70.0) overfit_score, overfit_breakdown = compute_deployment_score(train, valid, test, overfit_wf, rqs_score=70.0) assert robust_score is not None assert overfit_score is not None assert robust_score > overfit_score assert robust_breakdown["gate_factor"] == 1.0 assert overfit_breakdown["gate_factor"] < 1.0 class TestComputeRobustnessGate: @staticmethod def _summary( *, positive_rate: float, median_63d: float, median_252d: float, worst_return: float, ) -> RobustnessMatrixSummary: return RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=12, overall_positive_window_rate_pct=positive_rate, overall_worst_return_pct=worst_return, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, mean_return_pct=5.0, median_return_pct=median_63d, worst_return_pct=-2.0, positive_window_rate_pct=positive_rate, mean_max_drawdown_pct=4.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=3, mean_return_pct=12.0, median_return_pct=median_252d, worst_return_pct=1.0, positive_window_rate_pct=100.0, mean_max_drawdown_pct=6.0, ), ], ) def test_all_four_checks_pass(self): factor, breakdown = compute_robustness_gate( self._summary(positive_rate=70.0, median_63d=4.0, median_252d=9.0, worst_return=-5.0) ) assert factor == 1.0 assert sum(breakdown.values()) == 4.0 def test_three_checks_pass(self): factor, breakdown = compute_robustness_gate( self._summary(positive_rate=70.0, median_63d=4.0, median_252d=9.0, worst_return=-20.0) ) assert factor == 0.85 assert sum(breakdown.values()) == 3.0 def test_two_checks_pass(self): factor, breakdown = compute_robustness_gate( self._summary(positive_rate=70.0, median_63d=1.0, median_252d=9.0, worst_return=-20.0) ) assert factor == 0.65 assert sum(breakdown.values()) == 2.0 def test_one_check_pass(self): factor, breakdown = compute_robustness_gate( self._summary(positive_rate=50.0, median_63d=1.0, median_252d=9.0, worst_return=-20.0) ) assert factor == 0.40 assert sum(breakdown.values()) == 1.0 def test_zero_checks_pass(self): factor, breakdown = compute_robustness_gate( self._summary(positive_rate=50.0, median_63d=1.0, median_252d=5.0, worst_return=-20.0) ) assert factor == 0.20 assert sum(breakdown.values()) == 0.0 # --------------------------------------------------------------------------- # OOT-specific functions # --------------------------------------------------------------------------- class TestComputeOotPositiveRate63Plus: @staticmethod def _summary_with_21d( *, pos_21d: float = 30.0, pos_63d: float = 54.5, pos_252d: float = 61.5, cnt_21d: int = 24, cnt_63d: int = 22, cnt_252d: int = 13, ) -> RobustnessMatrixSummary: return RobustnessMatrixSummary( horizons_days=[21, 63, 252], step_days=21, overall_window_count=cnt_21d + cnt_63d + cnt_252d, overall_positive_window_rate_pct=40.0, overall_worst_return_pct=-12.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=21, window_count=cnt_21d, mean_return_pct=0.5, median_return_pct=0.1, worst_return_pct=-3.0, positive_window_rate_pct=pos_21d, mean_max_drawdown_pct=2.0, ), RobustnessHorizonSummary( horizon_days=63, window_count=cnt_63d, mean_return_pct=2.0, median_return_pct=1.33, worst_return_pct=-5.0, positive_window_rate_pct=pos_63d, mean_max_drawdown_pct=4.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=cnt_252d, mean_return_pct=8.0, median_return_pct=6.72, worst_return_pct=-12.0, positive_window_rate_pct=pos_252d, mean_max_drawdown_pct=10.0, ), ], ) def test_excludes_21d_windows(self): s = self._summary_with_21d(pos_21d=10.0, pos_63d=54.5, pos_252d=61.5) rate = _compute_oot_positive_rate_63plus(s) # (54.5*22 + 61.5*13) / (22+13) = (1199 + 799.5) / 35 ≈ 57.1 assert rate is not None assert 57.0 <= rate <= 57.2 def test_63d_only(self): s = RobustnessMatrixSummary( horizons_days=[63], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=70.0, overall_worst_return_pct=-5.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=10, mean_return_pct=3.0, median_return_pct=2.5, worst_return_pct=-2.0, positive_window_rate_pct=70.0, mean_max_drawdown_pct=3.0, ), ], ) assert _compute_oot_positive_rate_63plus(s) == 70.0 def test_no_qualifying_horizons(self): s = RobustnessMatrixSummary( horizons_days=[21], step_days=21, overall_window_count=5, overall_positive_window_rate_pct=40.0, overall_worst_return_pct=-8.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=21, window_count=5, mean_return_pct=0.5, median_return_pct=0.1, worst_return_pct=-3.0, positive_window_rate_pct=40.0, mean_max_drawdown_pct=2.0, ), ], ) assert _compute_oot_positive_rate_63plus(s) is None class TestComputeOotRobustnessGate: @staticmethod def _summary( *, pos_63d: float, pos_252d: float, median_63d: float, median_252d: float, worst_return: float, ) -> RobustnessMatrixSummary: return RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=50.0, overall_worst_return_pct=worst_return, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, mean_return_pct=2.0, median_return_pct=median_63d, worst_return_pct=-5.0, positive_window_rate_pct=pos_63d, mean_max_drawdown_pct=4.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=8.0, median_return_pct=median_252d, worst_return_pct=-1.0, positive_window_rate_pct=pos_252d, mean_max_drawdown_pct=6.0, ), ], ) def test_all_four_pass_stress(self): # 63d+ rate: (60*6 + 50*4)/10 = 56% >= 50 ✓ factor, breakdown = compute_oot_robustness_gate( self._summary(pos_63d=60.0, pos_252d=50.0, median_63d=1.0, median_252d=4.0, worst_return=-10.0) ) assert factor == 1.0 assert sum(breakdown.values()) == 4.0 def test_three_pass(self): # 63d+ rate: (60*6 + 50*4)/10 = 56% >= 50 ✓, 63d med 1.0 >= 0.5 ✓, # 252d med 4.0 >= 3.0 ✓, worst -16 < -15 ✗ factor, _ = compute_oot_robustness_gate( self._summary(pos_63d=60.0, pos_252d=50.0, median_63d=1.0, median_252d=4.0, worst_return=-16.0) ) assert factor == 0.85 def test_zero_pass_stress(self): # 63d+ rate: (40*6 + 30*4)/10 = 36% < 50 ✗, 63d med 0.2 < 0.5 ✗, # 252d med 1.0 < 3.0 ✗, worst -20 < -15 ✗ factor, _ = compute_oot_robustness_gate( self._summary(pos_63d=40.0, pos_252d=30.0, median_63d=0.2, median_252d=1.0, worst_return=-20.0) ) assert factor == 0.20 def test_none_summary(self): factor, breakdown = compute_oot_robustness_gate(None) assert factor == 1.0 assert breakdown == {} def test_sparse_zero_trade_summary_is_neutral(self): summary = RobustnessMatrixSummary( horizons_days=[21, 63, 252], step_days=21, overall_window_count=12, overall_positive_window_rate_pct=0.0, overall_worst_return_pct=0.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=21, window_count=5, mean_return_pct=0.0, median_return_pct=0.0, worst_return_pct=0.0, positive_window_rate_pct=0.0, mean_max_drawdown_pct=0.0, ), RobustnessHorizonSummary( horizon_days=63, window_count=4, mean_return_pct=0.0, median_return_pct=0.0, worst_return_pct=0.0, positive_window_rate_pct=0.0, mean_max_drawdown_pct=0.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=3, mean_return_pct=0.0, median_return_pct=0.0, worst_return_pct=0.0, positive_window_rate_pct=0.0, mean_max_drawdown_pct=0.0, ), ], ) factor, breakdown = compute_oot_robustness_gate(summary) assert factor == 1.0 assert breakdown["rb_gate_sparse_no_trade"] == 1.0 def test_main_gate_unchanged(self): """Ensure compute_robustness_gate thresholds are not affected.""" s = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=70.0, overall_worst_return_pct=-5.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, mean_return_pct=4.0, median_return_pct=4.0, worst_return_pct=-1.0, positive_window_rate_pct=80.0, mean_max_drawdown_pct=3.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=12.0, median_return_pct=9.0, worst_return_pct=1.0, positive_window_rate_pct=100.0, mean_max_drawdown_pct=5.0, ), ], ) factor, _ = compute_robustness_gate(s) assert factor == 1.0 class TestComputeOotRobustnessQuality: def test_strong_stress_test_survivor(self): s = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=35, overall_positive_window_rate_pct=55.0, overall_worst_return_pct=-12.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=22, mean_return_pct=2.0, median_return_pct=1.33, worst_return_pct=-5.0, positive_window_rate_pct=54.5, mean_max_drawdown_pct=4.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=13, mean_return_pct=8.0, median_return_pct=6.72, worst_return_pct=-12.0, positive_window_rate_pct=61.5, mean_max_drawdown_pct=10.0, ), ], ) quality, breakdown = compute_oot_robustness_quality(s) assert quality is not None assert quality > 75.0 assert "rb_quality_positive_rate" in breakdown def test_none_returns_none(self): quality, breakdown = compute_oot_robustness_quality(None) assert quality is None assert breakdown == {} def test_sparse_zero_trade_summary_returns_none(self): summary = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=0.0, overall_worst_return_pct=0.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, mean_return_pct=0.0, median_return_pct=0.0, worst_return_pct=0.0, positive_window_rate_pct=0.0, mean_max_drawdown_pct=0.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=0.0, median_return_pct=0.0, worst_return_pct=0.0, positive_window_rate_pct=0.0, mean_max_drawdown_pct=0.0, ), ], ) quality, breakdown = compute_oot_robustness_quality(summary) assert quality is None assert breakdown["rb_quality_sparse_no_trade"] == 1.0 def test_weak_stress_test(self): s = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=35, overall_positive_window_rate_pct=35.0, overall_worst_return_pct=-22.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=22, mean_return_pct=-1.0, median_return_pct=-0.76, worst_return_pct=-8.0, positive_window_rate_pct=45.5, mean_max_drawdown_pct=6.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=13, mean_return_pct=-2.0, median_return_pct=-0.92, worst_return_pct=-22.0, positive_window_rate_pct=46.2, mean_max_drawdown_pct=18.0, ), ], ) quality, _ = compute_oot_robustness_quality(s) assert quality is not None assert quality < 65.0 def test_refresh_public_scores_recomputes_stale_scores(tmp_path: Path): journal_path = tmp_path / "improvement_journal.jsonl" train = SplitResult(run_id="bt_train", trade_count=20, total_return_pct=20.0, win_rate=0.7) valid = SplitResult(run_id="bt_valid", trade_count=15, total_return_pct=10.0, win_rate=0.6) test = SplitResult( run_id="bt_test", trade_count=24, total_return_pct=12.0, win_rate=0.6, days_in_market_pct=55.0, ) summary = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=8, test_aggregate=WalkForwardAggregate( mean_return_pct=10.0, median_return_pct=8.0, worst_return_pct=1.0, positive_fold_rate_pct=87.5, mean_profit_factor=1.8, mean_max_drawdown_pct=5.0, mean_trade_count=12.0, mean_win_rate=0.55, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=20.0, fold_return_cv=0.3, ), ) robustness = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=90.0, overall_worst_return_pct=-2.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, mean_return_pct=4.0, median_return_pct=3.5, worst_return_pct=-1.0, positive_window_rate_pct=83.3, mean_max_drawdown_pct=3.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=18.0, median_return_pct=12.0, worst_return_pct=4.0, positive_window_rate_pct=100.0, mean_max_drawdown_pct=5.0, ), ], ) stale_entry = JournalEntry( entry_id="IMP-0001", timestamp="2026-03-22T00:00:00+00:00", experiment_name="return_max_long_v1.200", hypothesis="stale", results={"train": train, "valid": valid, "test": test}, walk_forward_summary=summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=robustness, sqs_score=999.0, sqs_breakdown={"old": 1.0}, sqs_v2_score=999.0, sqs_v2_breakdown={"old": 1.0}, rqs_score=1.0, wfqs_score=1.0, wfqs_v2_score=1.0, deployment_score=1.0, ) append_journal_entry(journal_path, stale_entry) updated = refresh_public_scores(journal_path, selector=lambda e: e.experiment_name.endswith("v1.200")) assert updated == 1 refreshed = load_journal(journal_path)[0] expected_sqs, expected_breakdown, _ = compute_public_sqs( train, valid, test, walk_forward_summary=summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=robustness, ) expected_stress_sqs, expected_stress_breakdown, _ = compute_public_sqs_v2( train, valid, test, walk_forward_summary=summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=robustness, ) assert refreshed.sqs_score == expected_sqs assert refreshed.sqs_breakdown == expected_breakdown assert refreshed.stress_sqs_score == expected_stress_sqs assert refreshed.stress_sqs_breakdown == expected_stress_breakdown assert refreshed.sqs_score != 999.0 assert refreshed.sqs_v2_score != 999.0 # --------------------------------------------------------------------------- # build_split_result # --------------------------------------------------------------------------- def test_build_split_result(): m = MetricsBundle( trade_count=50, profit_factor=1.2, total_return_pct=2.5, win_rate=0.55, max_drawdown_pct=3.0, sharpe_ratio=0.8, monthly_win_rate=0.60, equity_curve_r_squared=0.40, avg_gross_exposure_pct=18.5, avg_net_exposure_pct=-6.5, days_in_market_pct=27.0, ) sr = build_split_result("test", "bt_run123", m) assert sr.run_id == "bt_run123" assert sr.trade_count == 50 assert sr.profit_factor == 1.2 assert sr.total_return_pct == 2.5 assert sr.avg_gross_exposure_pct == 18.5 assert sr.avg_net_exposure_pct == -6.5 assert sr.days_in_market_pct == 27.0 # --------------------------------------------------------------------------- # Journal I/O # --------------------------------------------------------------------------- class TestJournalIO: def test_append_and_load(self, tmp_path): journal_path = tmp_path / "journal.jsonl" entry = JournalEntry( entry_id="IMP-0001", timestamp="2026-03-16T12:00:00", experiment_name="test_exp_1", hypothesis="Test hypothesis", sqs_score=55.0, sqs_breakdown={"profitability": 60.0, "risk": 50.0, "consistency": 55.0, "robustness": 50.0}, verdict="better", ) append_journal_entry(journal_path, entry) entries = load_journal(journal_path) assert len(entries) == 1 assert entries[0].entry_id == "IMP-0001" assert entries[0].experiment_name == "test_exp_1" assert entries[0].sqs_score == 55.0 def test_multiple_entries(self, tmp_path): journal_path = tmp_path / "journal.jsonl" for i in range(3): entry = JournalEntry( entry_id=f"IMP-{i+1:04d}", timestamp=f"2026-03-{16+i}T12:00:00", experiment_name=f"exp_{i}", hypothesis=f"Hypothesis {i}", sqs_score=float(40 + i * 10), ) append_journal_entry(journal_path, entry) entries = load_journal(journal_path) assert len(entries) == 3 assert entries[2].sqs_score == 60.0 def test_get_next_entry_id(self, tmp_path): journal_path = tmp_path / "journal.jsonl" assert get_next_entry_id(journal_path) == "IMP-0001" entry = JournalEntry( entry_id="IMP-0001", timestamp="2026-03-16T12:00:00", experiment_name="exp_1", hypothesis="h", ) append_journal_entry(journal_path, entry) assert get_next_entry_id(journal_path) == "IMP-0002" def test_journal_lock_serializes_concurrent_writers(self, tmp_path): journal_path = tmp_path / "journal.jsonl" ctx = multiprocessing.get_context("spawn") payloads = [ (str(journal_path), "exp_a"), (str(journal_path), "exp_b"), (str(journal_path), "exp_c"), ] with ctx.Pool(processes=3) as pool: ids = pool.map(_write_locked_journal_entry, payloads) assert sorted(ids) == ["IMP-0001", "IMP-0002", "IMP-0003"] assert [entry.entry_id for entry in load_journal(journal_path)] == [ "IMP-0001", "IMP-0002", "IMP-0003", ] def test_load_empty(self, tmp_path): journal_path = tmp_path / "nonexistent.jsonl" entries = load_journal(journal_path) assert entries == [] # --------------------------------------------------------------------------- # check_duplicate # --------------------------------------------------------------------------- class TestCheckDuplicate: def test_finds_duplicates(self, tmp_path): journal_path = tmp_path / "journal.jsonl" for name in ["exp_a", "exp_b", "exp_a"]: entry = JournalEntry( entry_id=get_next_entry_id(journal_path), timestamp="2026-03-16T12:00:00", experiment_name=name, hypothesis="h", ) append_journal_entry(journal_path, entry) dupes = check_duplicate(journal_path, "exp_a") assert len(dupes) == 2 def test_no_duplicates(self, tmp_path): journal_path = tmp_path / "journal.jsonl" entry = JournalEntry( entry_id="IMP-0001", timestamp="2026-03-16T12:00:00", experiment_name="exp_a", hypothesis="h", ) append_journal_entry(journal_path, entry) dupes = check_duplicate(journal_path, "exp_z") assert len(dupes) == 0 # --------------------------------------------------------------------------- # rebuild_registry # --------------------------------------------------------------------------- class TestRebuildRegistry: @staticmethod def _validated_summary_triplet() -> tuple[WalkForwardSummary, RobustnessMatrixSummary, RobustnessMatrixSummary]: wfv = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=8, test_aggregate=WalkForwardAggregate( mean_return_pct=12.0, median_return_pct=8.0, worst_return_pct=1.0, positive_fold_rate_pct=87.5, mean_profit_factor=1.9, mean_max_drawdown_pct=4.0, mean_trade_count=18.0, mean_win_rate=0.56, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=24.0, worst_train_test_return_gap_pct=30.0, fold_return_cv=0.5, ), ) robustness = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=80.0, overall_worst_return_pct=-2.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, mean_return_pct=4.5, median_return_pct=3.5, worst_return_pct=-1.0, positive_window_rate_pct=83.3, mean_max_drawdown_pct=3.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=16.0, median_return_pct=12.0, worst_return_pct=4.0, positive_window_rate_pct=100.0, mean_max_drawdown_pct=5.0, ), ], ) oot = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=72.0, overall_worst_return_pct=-4.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, mean_return_pct=3.0, median_return_pct=2.5, worst_return_pct=-2.0, positive_window_rate_pct=66.7, mean_max_drawdown_pct=3.2, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=10.0, median_return_pct=8.0, worst_return_pct=1.5, positive_window_rate_pct=75.0, mean_max_drawdown_pct=5.2, ), ], ) return wfv, robustness, oot def test_registry_and_leaderboard(self, tmp_path): journal_path = tmp_path / "journal.jsonl" registry_path = tmp_path / "registry.json" leaderboard_path = tmp_path / "LEADERBOARD.md" wfv, robustness, oot = self._validated_summary_triplet() # Create entries with different SQS scores for i, (name, sqs) in enumerate([("exp_low", 30.0), ("exp_high", 70.0), ("exp_mid", 50.0)]): test_result = SplitResult( run_id=f"bt_{name}", trade_count=60, profit_factor=1.0 + i * 0.2, total_return_pct=float(i), win_rate=0.5, monthly_win_rate=0.6, equity_curve_r_squared=0.5, avg_gross_exposure_pct=5.0, avg_net_exposure_pct=-1.0, days_in_market_pct=60.0, ) valid_result = SplitResult( run_id=f"bt_{name}_valid", trade_count=25, profit_factor=1.0 + i * 0.1, total_return_pct=float(i) / 2, win_rate=0.45, monthly_win_rate=0.5, equity_curve_r_squared=0.3, avg_gross_exposure_pct=4.0, avg_net_exposure_pct=-0.5, days_in_market_pct=45.0, ) entry = JournalEntry( entry_id=f"IMP-{i+1:04d}", timestamp=f"2026-03-{16+i}T12:00:00", experiment_name=name, hypothesis=f"h{i}", sqs_score=sqs, promotion_score=sqs - 5, unified_score=sqs - 10, results={"test": test_result, "valid": valid_result}, walk_forward_summary=wfv, robustness_matrix_summary=robustness, out_of_time_robustness_summary=oot, verdict="better" if sqs > 50 else "worse", ) append_journal_entry(journal_path, entry) registry = rebuild_registry(journal_path, registry_path, leaderboard_path) # Sorted by public SQS descending assert len(registry.entries) == 3 assert registry.entries[0].sqs_score is not None assert registry.entries[-1].sqs_score is not None assert registry.entries[0].sqs_score >= registry.entries[1].sqs_score >= registry.entries[2].sqs_score assert registry.entries[0].sqs_v2_score is not None assert registry.entries[0].promotion_score is not None assert registry.entries[0].sqs_score < 70.0 # Files exist assert registry_path.exists() assert leaderboard_path.exists() # Leaderboard contains table lb_text = leaderboard_path.read_text() assert "exp_high" in lb_text assert "exp_low" in lb_text assert "| # |" in lb_text assert "| # | ID | Experiment | SQS |" in lb_text def test_registry_skips_legacy_overlay_entries(self, tmp_path): journal_path = tmp_path / "journal.jsonl" registry_path = tmp_path / "registry.json" leaderboard_path = tmp_path / "LEADERBOARD.md" entry = JournalEntry( entry_id="IMP-0001", timestamp="2026-03-25T22:00:00+00:00", experiment_name="return_book_overlay_v3", hypothesis="overlay", tags=["overlay"], ) append_journal_entry(journal_path, entry) registry = rebuild_registry(journal_path, registry_path, leaderboard_path) visible = filter_registry_entries(registry.entries) assert len(visible) == 0 assert registry.entries == [] lb_text = leaderboard_path.read_text() assert "return_book_overlay_v3" not in lb_text assert "Default view excludes retired legacy PEAD" in lb_text def test_registry_preserves_walk_forward_summary(self, tmp_path): journal_path = tmp_path / "journal.jsonl" registry_path = tmp_path / "registry.json" leaderboard_path = tmp_path / "LEADERBOARD.md" entry = JournalEntry( entry_id="IMP-0001", timestamp="2026-03-19T10:00:00+00:00", experiment_name="return_max_long_v999_wfv", hypothesis="wfv", results={ "valid": SplitResult(run_id="bt_valid", trade_count=5, total_return_pct=5.0), "test": SplitResult(run_id="bt_test", trade_count=5, total_return_pct=7.0), }, walk_forward_summary=WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=2, folds=[ WalkForwardFoldResult( fold_index=0, train_start="2024-01-01", train_end="2024-12-31", test_start="2025-01-01", test_end="2025-03-31", train_run_id="wf_train_0", test_run_id="wf_test_0", train_metrics=SplitResult(run_id="wf_train_0", trade_count=3, total_return_pct=10.0), test_metrics=SplitResult(run_id="wf_test_0", trade_count=2, total_return_pct=4.0), ), ], train_aggregate=WalkForwardAggregate(mean_return_pct=12.5), test_aggregate=WalkForwardAggregate(mean_return_pct=6.5, median_return_pct=6.5, worst_return_pct=4.0, positive_fold_rate_pct=100.0), gap_stats=WalkForwardGapStats(mean_train_test_return_gap_pct=6.0, worst_train_test_return_gap_pct=8.0), ), ) append_journal_entry(journal_path, entry) registry = rebuild_registry(journal_path, registry_path, leaderboard_path) assert registry.entries[0].walk_forward_summary is not None assert registry.entries[0].walk_forward_summary.fold_count == 2 registry_payload = json.loads(registry_path.read_text()) assert registry_payload["entries"][0]["walk_forward_summary"]["test_aggregate"]["mean_return_pct"] == 6.5 def test_attach_walk_forward_summary_backfills_scores(self, tmp_path): journal_path = tmp_path / "journal.jsonl" registry_path = tmp_path / "registry.json" leaderboard_path = tmp_path / "LEADERBOARD.md" entry = JournalEntry( entry_id="IMP-0001", timestamp="2026-03-19T10:00:00+00:00", experiment_name="return_max_long_v999_wfv", hypothesis="wfv", results={ "train": SplitResult(run_id="bt_train", trade_count=5, total_return_pct=8.0), "valid": SplitResult(run_id="bt_valid", trade_count=5, total_return_pct=6.0), "test": SplitResult(run_id="bt_test", trade_count=5, total_return_pct=7.0), }, ) append_journal_entry(journal_path, entry) summary = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=2, folds=[ WalkForwardFoldResult( fold_index=0, train_start="2024-01-01", train_end="2024-12-31", test_start="2025-01-01", test_end="2025-03-31", train_run_id="wf_train_0", test_run_id="wf_test_0", train_metrics=SplitResult(run_id="wf_train_0", trade_count=3, total_return_pct=10.0, profit_factor=1.5), test_metrics=SplitResult(run_id="wf_test_0", trade_count=2, total_return_pct=4.0, profit_factor=1.3), ), WalkForwardFoldResult( fold_index=1, train_start="2024-04-01", train_end="2025-03-31", test_start="2025-04-01", test_end="2025-06-30", train_run_id="wf_train_1", test_run_id="wf_test_1", train_metrics=SplitResult(run_id="wf_train_1", trade_count=4, total_return_pct=11.0, profit_factor=1.4), test_metrics=SplitResult(run_id="wf_test_1", trade_count=3, total_return_pct=5.0, profit_factor=1.2), ), ], train_aggregate=WalkForwardAggregate(mean_return_pct=10.5), test_aggregate=WalkForwardAggregate( mean_return_pct=4.5, median_return_pct=4.5, worst_return_pct=4.0, positive_fold_rate_pct=100.0, mean_profit_factor=1.25, mean_max_drawdown_pct=2.5, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=6.0, worst_train_test_return_gap_pct=6.0, ), ) updated = attach_walk_forward_summary(journal_path, "return_max_long_v999_wfv", summary) assert updated.walk_forward_summary is not None assert updated.wfqs_score is not None assert updated.deployment_score is not None registry = rebuild_registry(journal_path, registry_path, leaderboard_path) assert registry.entries[0].walk_forward_summary is not None assert registry.entries[0].wfqs_score is not None assert registry.entries[0].deployment_score is not None def test_attach_robustness_summary_updates_existing_entry(self, tmp_path): journal_path = tmp_path / "journal.jsonl" registry_path = tmp_path / "registry.json" leaderboard_path = tmp_path / "LEADERBOARD.md" entry = JournalEntry( entry_id="IMP-0001", timestamp="2026-03-19T10:00:00+00:00", experiment_name="return_max_long_v999_rb", hypothesis="rb", results={ "train": SplitResult(run_id="bt_train", trade_count=5, total_return_pct=12.0), "valid": SplitResult(run_id="bt_valid", trade_count=5, total_return_pct=6.0), "test": SplitResult(run_id="bt_test", trade_count=5, total_return_pct=7.0), }, walk_forward_summary=WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=2, train_aggregate=WalkForwardAggregate(mean_return_pct=10.0), test_aggregate=WalkForwardAggregate( mean_return_pct=6.5, median_return_pct=6.5, worst_return_pct=4.0, positive_fold_rate_pct=100.0, mean_profit_factor=1.5, mean_max_drawdown_pct=3.0, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=8.0, worst_train_test_return_gap_pct=10.0, ), ), ) append_journal_entry(journal_path, entry) summary = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=70.0, overall_worst_return_pct=-15.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, mean_return_pct=4.0, median_return_pct=4.0, worst_return_pct=-2.0, positive_window_rate_pct=80.0, mean_max_drawdown_pct=2.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=2, mean_return_pct=10.0, median_return_pct=9.0, worst_return_pct=3.0, positive_window_rate_pct=100.0, mean_max_drawdown_pct=4.0, ), ], ) updated = attach_robustness_summary(journal_path, "return_max_long_v999_rb", summary) assert updated.robustness_matrix_summary is not None assert updated.robustness_matrix_summary.overall_window_count == 10 registry = rebuild_registry(journal_path, registry_path, leaderboard_path) assert registry.entries[0].robustness_matrix_summary is not None assert registry.entries[0].deployment_score is not None expected_sqs, _, expected_source = compute_public_sqs( updated.results.get("train"), updated.results.get("valid"), updated.results.get("test"), walk_forward_summary=updated.walk_forward_summary, robustness_matrix_summary=updated.robustness_matrix_summary, rqs_score=registry.entries[0].rqs_score, wfqs_score=registry.entries[0].wfqs_score, deployment_score=registry.entries[0].deployment_score, ) assert expected_source == "pending_validation" assert expected_sqs is None assert registry.entries[0].sqs_score is None def test_attach_out_of_time_robustness_summary_updates_existing_entry(self, tmp_path): journal_path = tmp_path / "journal.jsonl" registry_path = tmp_path / "registry.json" leaderboard_path = tmp_path / "LEADERBOARD.md" entry = JournalEntry( entry_id="IMP-0001", timestamp="2026-03-19T10:00:00+00:00", experiment_name="return_max_long_v999_oot", hypothesis="oot", results={ "train": SplitResult(run_id="bt_train", trade_count=5, total_return_pct=12.0), "valid": SplitResult(run_id="bt_valid", trade_count=5, total_return_pct=6.0), "test": SplitResult(run_id="bt_test", trade_count=5, total_return_pct=7.0), }, walk_forward_summary=WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=2, train_aggregate=WalkForwardAggregate(mean_return_pct=10.0), test_aggregate=WalkForwardAggregate( mean_return_pct=6.5, median_return_pct=6.5, worst_return_pct=4.0, positive_fold_rate_pct=100.0, mean_profit_factor=1.5, mean_max_drawdown_pct=3.0, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=8.0, worst_train_test_return_gap_pct=10.0, ), ), robustness_matrix_summary=RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=80.0, overall_worst_return_pct=-2.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, mean_return_pct=4.0, median_return_pct=4.0, worst_return_pct=-2.0, positive_window_rate_pct=80.0, mean_max_drawdown_pct=2.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=2, mean_return_pct=10.0, median_return_pct=9.0, worst_return_pct=3.0, positive_window_rate_pct=100.0, mean_max_drawdown_pct=4.0, ), ], ), ) append_journal_entry(journal_path, entry) summary = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=60.0, overall_worst_return_pct=-15.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, mean_return_pct=2.0, median_return_pct=2.0, worst_return_pct=-2.0, positive_window_rate_pct=66.7, mean_max_drawdown_pct=3.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=2, mean_return_pct=7.0, median_return_pct=7.0, worst_return_pct=-1.0, positive_window_rate_pct=100.0, mean_max_drawdown_pct=5.0, ), ], ) updated = attach_out_of_time_robustness_summary(journal_path, "return_max_long_v999_oot", summary) assert updated.out_of_time_robustness_summary is not None registry = rebuild_registry(journal_path, registry_path, leaderboard_path) assert registry.entries[0].out_of_time_robustness_summary is not None def test_attach_walk_forward_summary_updates_existing_entry(self, tmp_path): journal_path = tmp_path / "journal.jsonl" append_journal_entry( journal_path, JournalEntry( entry_id="IMP-0001", timestamp="2026-03-19T12:00:00+00:00", experiment_name="return_max_long_v334_v326_material_compact", hypothesis="attach wfv", results={ "train": SplitResult(run_id="bt_train", trade_count=10, total_return_pct=55.0), "valid": SplitResult(run_id="bt_valid", trade_count=4, total_return_pct=12.0), "test": SplitResult(run_id="bt_test", trade_count=5, total_return_pct=18.0), }, rqs_score=70.0, ), ) summary = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=3, folds=[ WalkForwardFoldResult( fold_index=0, train_start="2024-01-01", train_end="2024-12-31", test_start="2025-01-01", test_end="2025-03-31", train_run_id="wf_train_0", test_run_id="wf_test_0", train_metrics=SplitResult(run_id="wf_train_0", trade_count=3, total_return_pct=20.0), test_metrics=SplitResult(run_id="wf_test_0", trade_count=2, total_return_pct=8.0), ), ], train_aggregate=WalkForwardAggregate(mean_return_pct=25.0), test_aggregate=WalkForwardAggregate( mean_return_pct=9.0, median_return_pct=8.0, worst_return_pct=2.0, positive_fold_rate_pct=100.0, mean_profit_factor=2.2, mean_max_drawdown_pct=3.0, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=16.0, worst_train_test_return_gap_pct=22.0, ), ) updated = attach_walk_forward_summary(journal_path, "return_max_long_v334_v326_material_compact", summary) assert updated.walk_forward_summary is not None assert updated.walk_forward_summary.fold_count == 3 assert updated.wfqs_score is not None assert updated.deployment_score is not None persisted = load_journal(journal_path) assert len(persisted) == 1 assert persisted[0].walk_forward_summary is not None assert persisted[0].wfqs_score == updated.wfqs_score assert persisted[0].deployment_score == updated.deployment_score def test_retired_short_core_and_legacy_pead_are_hidden_by_default(self, tmp_path): journal_path = tmp_path / "journal.jsonl" registry_path = tmp_path / "registry.json" leaderboard_path = tmp_path / "LEADERBOARD.md" wfv, robustness, oot = self._validated_summary_triplet() test_result = SplitResult( run_id="bt_test", trade_count=60, profit_factor=1.8, total_return_pct=1.2, win_rate=0.55, monthly_win_rate=0.60, equity_curve_r_squared=0.5, avg_gross_exposure_pct=5.0, avg_net_exposure_pct=2.0, days_in_market_pct=60.0, ) valid_result = SplitResult( run_id="bt_valid", trade_count=12, profit_factor=1.4, total_return_pct=0.6, win_rate=0.5, monthly_win_rate=0.5, equity_curve_r_squared=0.2, avg_gross_exposure_pct=4.0, avg_net_exposure_pct=1.5, days_in_market_pct=40.0, ) append_journal_entry( journal_path, JournalEntry( entry_id="IMP-0001", timestamp="2026-03-18T12:00:00", experiment_name="pead_midcap_step56_short_core_macro_block_crashcap_gap10_interleave_longtrend25", hypothesis="legacy", sqs_score=51.3, results={"test": test_result, "valid": valid_result}, walk_forward_summary=wfv, robustness_matrix_summary=robustness, out_of_time_robustness_summary=oot, tags=["pead", "short_core"], ), ) append_journal_entry( journal_path, JournalEntry( entry_id="IMP-0002", timestamp="2026-03-18T13:00:00", experiment_name="return_max_long_v32_same_day_longer_tail", hypothesis="active", sqs_score=14.9, results={"test": test_result, "valid": valid_result}, walk_forward_summary=wfv, robustness_matrix_summary=robustness, out_of_time_robustness_summary=oot, tags=["return", "max", "long"], ), ) registry = rebuild_registry(journal_path, registry_path, leaderboard_path) assert len(registry.entries) == 2 assert classify_strategy_family(registry.entries[0].experiment_name) in {"short_core", "return_max_long"} retired_hidden = filter_registry_entries(registry.entries, include_retired=False) assert [entry.experiment_name for entry in retired_hidden] == ["return_max_long_v32_same_day_longer_tail"] assert retired_hidden[0].is_retired is False retired_entry = next(entry for entry in registry.entries if entry.experiment_name.startswith("pead_midcap_step56")) assert retired_entry.is_retired is True assert retired_entry.strategy_family == "short_core" lb_text = leaderboard_path.read_text() assert "return_max_long_v32_same_day_longer_tail" in lb_text assert "pead_midcap_step56_short_core_macro_block_crashcap_gap10_interleave_longtrend25" not in lb_text assert "--include-retired" in lb_text def test_manifest_exact_branches_are_structurally_retired(self, tmp_path, monkeypatch): from libs.backtest import tracker as tracker_module cfg_dir = tmp_path / "configs" / "experiments" cfg_dir.mkdir(parents=True) (cfg_dir / "return_max_long_tmp_exact_probe.json").write_text( json.dumps( { "experiment_name": "return_max_long_tmp_exact_probe", "strategy_engines": [ {"engine_id": "next_open_long_probe_exact"}, ], } ) ) tracker_module._load_manifest_json.cache_clear() monkeypatch.setattr(tracker_module, "_EXPERIMENTS_DIR", cfg_dir) assert classify_strategy_family("return_max_long_tmp_exact_probe") == "exact_pocket_return_max_long" assert classify_strategy_family("return_max_long_clean_restart_v326") == "return_max_long" tracker_module._load_manifest_json.cache_clear() def test_manifest_named_micro_branches_are_structurally_retired_only_when_enabled(self, tmp_path, monkeypatch): from libs.backtest import tracker as tracker_module cfg_dir = tmp_path / "configs" / "experiments" cfg_dir.mkdir(parents=True) (cfg_dir / "return_max_long_tmp_named_micro.json").write_text( json.dumps( { "experiment_name": "return_max_long_tmp_named_micro", "strategy_engines": [ {"engine_id": "next_open_long_unknown_guidance_apld_micro", "enabled": True}, ], } ) ) (cfg_dir / "return_max_long_tmp_named_micro_disabled.json").write_text( json.dumps( { "experiment_name": "return_max_long_tmp_named_micro_disabled", "strategy_engines": [ {"engine_id": "next_open_long_unknown_guidance_apld_micro", "enabled": False}, ], } ) ) tracker_module._load_manifest_json.cache_clear() monkeypatch.setattr(tracker_module, "_EXPERIMENTS_DIR", cfg_dir) assert classify_strategy_family("return_max_long_tmp_named_micro") == "named_micro_return_max_long" assert classify_strategy_family("return_max_long_tmp_named_micro_disabled") == "named_micro_return_max_long" tracker_module._load_manifest_json.cache_clear() def test_scan_runs_for_experiment_recurses_and_prefers_latest_split_run(self, tmp_path): runs_dir = tmp_path / "runs" experiment_name = "return_max_long_v1.10" def _write_run(batch: str, run_name: str, split: str, started_at: str, total_return_pct: float) -> None: run_dir = runs_dir / batch / run_name (run_dir / "metrics").mkdir(parents=True) (run_dir / "manifest.json").write_text(json.dumps({"experiment_name": experiment_name})) (run_dir / "metadata.json").write_text( json.dumps( { "run_id": run_name, "split_name": split, "started_at": started_at, } ) ) metrics = MetricsBundle( trade_count=12, profit_factor=1.8, total_return_pct=total_return_pct, win_rate=0.58, monthly_win_rate=0.6, equity_curve_r_squared=0.4, max_drawdown_pct=3.0, sharpe_ratio=1.2, avg_gross_exposure_pct=11.0, avg_net_exposure_pct=11.0, days_in_market_pct=22.0, ) (run_dir / "metrics" / "metrics_summary.json").write_text(metrics.model_dump_json()) _write_run("batch_old", "bt_old_train", "train", "2026-03-20T01:00:00+00:00", 10.0) _write_run("batch_new", "bt_new_train", "train", "2026-03-21T01:00:00+00:00", 25.0) _write_run("batch_new", "bt_new_valid", "valid", "2026-03-21T01:05:00+00:00", 15.0) _write_run("batch_new", "bt_new_test", "test", "2026-03-21T01:10:00+00:00", 18.0) results = scan_runs_for_experiment(runs_dir, experiment_name) assert set(results) == {"train", "valid", "test"} assert results["train"][0] == "bt_new_train" assert results["train"][1].total_return_pct == 25.0 def test_scan_runs_for_experiment_ignores_walk_forward_pseudo_splits(self, tmp_path): runs_dir = tmp_path / "runs" experiment_name = "return_max_long_v1.128" def _write_run(batch: str, run_name: str, split: str, started_at: str, total_return_pct: float) -> None: run_dir = runs_dir / batch / run_name (run_dir / "metrics").mkdir(parents=True) (run_dir / "manifest.json").write_text(json.dumps({"experiment_name": experiment_name})) (run_dir / "metadata.json").write_text( json.dumps( { "run_id": run_name, "split_name": split, "started_at": started_at, } ) ) metrics = MetricsBundle( trade_count=12, profit_factor=1.8, total_return_pct=total_return_pct, win_rate=0.58, monthly_win_rate=0.6, equity_curve_r_squared=0.4, max_drawdown_pct=3.0, sharpe_ratio=1.2, avg_gross_exposure_pct=11.0, avg_net_exposure_pct=11.0, days_in_market_pct=22.0, ) (run_dir / "metrics" / "metrics_summary.json").write_text(metrics.model_dump_json()) _write_run("fullsplit", "bt_train", "train", "2026-03-21T01:00:00+00:00", 10.0) _write_run("fullsplit", "bt_valid", "valid", "2026-03-21T01:05:00+00:00", 11.0) _write_run("fullsplit", "bt_test", "test", "2026-03-21T01:10:00+00:00", 12.0) _write_run("wfv", "bt_wf_train_00", "wf_train_00", "2026-03-21T02:00:00+00:00", 30.0) _write_run("wfv", "bt_wf_test_00", "wf_test_00", "2026-03-21T02:05:00+00:00", 20.0) results = scan_runs_for_experiment(runs_dir, experiment_name) assert set(results) == {"train", "valid", "test"} assert results["train"][0] == "bt_train" assert results["valid"][0] == "bt_valid" assert results["test"][0] == "bt_test" def test_scan_runs_for_experiment_ignores_snapshot_override_runs(self, tmp_path): runs_dir = tmp_path / "runs" experiment_name = "return_max_long_v8.32" manifest_payload = { "experiment_name": experiment_name, "dataset_snapshot_id": "midlarge-liquid-long-v1_bucketfix_full_audit_tier3", } def _write_run( batch: str, run_name: str, split: str, started_at: str, total_return_pct: float, *, resolved_snapshot_id: str, ) -> None: run_dir = runs_dir / batch / run_name (run_dir / "metrics").mkdir(parents=True) (run_dir / "manifest.json").write_text(json.dumps(manifest_payload)) (run_dir / "resolved_config.json").write_text( json.dumps({"dataset_snapshot_id": resolved_snapshot_id}) ) (run_dir / "metadata.json").write_text( json.dumps( { "run_id": run_name, "split_name": split, "started_at": started_at, } ) ) metrics = MetricsBundle( trade_count=12, profit_factor=1.8, total_return_pct=total_return_pct, win_rate=0.58, monthly_win_rate=0.6, equity_curve_r_squared=0.4, max_drawdown_pct=3.0, sharpe_ratio=1.2, avg_gross_exposure_pct=11.0, avg_net_exposure_pct=11.0, days_in_market_pct=22.0, ) (run_dir / "metrics" / "metrics_summary.json").write_text(metrics.model_dump_json()) _write_run( "fullsplit", "bt_train", "train", "2026-03-21T01:00:00+00:00", 10.0, resolved_snapshot_id="midlarge-liquid-long-v1_bucketfix_full_audit_tier3", ) _write_run( "fullsplit", "bt_valid", "valid", "2026-03-21T01:05:00+00:00", 11.0, resolved_snapshot_id="midlarge-liquid-long-v1_bucketfix_full_audit_tier3", ) _write_run( "fullsplit", "bt_test", "test", "2026-03-21T01:10:00+00:00", 12.0, resolved_snapshot_id="midlarge-liquid-long-v1_bucketfix_full_audit_tier3", ) _write_run( "common", "bt_common_train", "train", "2026-03-22T01:00:00+00:00", 99.0, resolved_snapshot_id="midlarge-liquid-long-v1_bucketfix_full_audit_tier3_merged", ) results = scan_runs_for_experiment(runs_dir, experiment_name) assert set(results) == {"train", "valid", "test"} assert results["train"][0] == "bt_train" assert results["train"][1].total_return_pct == 10.0 def test_scan_runs_for_experiment_accepts_canonical_snapshot_match(self, tmp_path): runs_dir = tmp_path / "runs" experiment_name = "return_max_long_v12.61" manifest_payload = { "experiment_name": experiment_name, "dataset_snapshot_id": "midlarge-liquid-long-v1_bucketfix_full_audit_tier3tech", } for split, started_at, total_return_pct in [ ("train", "2026-03-21T01:00:00+00:00", 10.0), ("valid", "2026-03-21T01:05:00+00:00", 11.0), ("test", "2026-03-21T01:10:00+00:00", 12.0), ]: run_dir = runs_dir / "batch" / f"bt_{split}" (run_dir / "metrics").mkdir(parents=True) (run_dir / "manifest.json").write_text(json.dumps(manifest_payload)) (run_dir / "resolved_config.json").write_text( json.dumps( { "dataset_snapshot_id": "midlarge-liquid-long-v1_bucketfix_full_audit_canonical", "requested_snapshot_id": "midlarge-liquid-long-v1_bucketfix_full_audit_tier3tech", "canonical_snapshot_id": "midlarge-liquid-long-v1_bucketfix_full_audit_canonical", } ) ) (run_dir / "metadata.json").write_text( json.dumps( { "run_id": f"bt_{split}", "split_name": split, "started_at": started_at, } ) ) metrics = MetricsBundle( trade_count=12, profit_factor=1.8, total_return_pct=total_return_pct, win_rate=0.58, monthly_win_rate=0.6, equity_curve_r_squared=0.4, max_drawdown_pct=3.0, sharpe_ratio=1.2, avg_gross_exposure_pct=11.0, avg_net_exposure_pct=11.0, days_in_market_pct=22.0, ) (run_dir / "metrics" / "metrics_summary.json").write_text(metrics.model_dump_json()) results = scan_runs_for_experiment(runs_dir, experiment_name) assert set(results) == {"train", "valid", "test"} assert results["test"][0] == "bt_test" def test_sync_official_manifests_auto_records_missing_manifest(self, tmp_path): journal_path = tmp_path / "journal.jsonl" configs_dir = tmp_path / "configs" / "experiments" runs_dir = tmp_path / "runs" configs_dir.mkdir(parents=True) experiment_name = "return_max_long_v1.10" (configs_dir / f"{experiment_name}.json").write_text( json.dumps( { "experiment_name": experiment_name, "description": "Clean baseline", "tags": ["return-max", "clean-lineage", "v1.10"], "strategy_engines": [ {"engine_id": "reaction_close_long_core", "enabled": True}, {"engine_id": "next_open_long_unknown_guidance_apld_micro", "enabled": False}, ], } ) ) def _write_run(split: str, total_return_pct: float) -> None: run_dir = runs_dir / "batch" / f"bt_{split}" (run_dir / "metrics").mkdir(parents=True) (run_dir / "manifest.json").write_text(json.dumps({"experiment_name": experiment_name})) (run_dir / "metadata.json").write_text( json.dumps( { "run_id": f"bt_{split}", "split_name": split, "started_at": f"2026-03-21T0{1 if split == 'train' else 2 if split == 'valid' else 3}:00:00+00:00", } ) ) metrics = MetricsBundle( trade_count=15, profit_factor=2.0, total_return_pct=total_return_pct, win_rate=0.6, monthly_win_rate=0.75, equity_curve_r_squared=0.5, max_drawdown_pct=3.0, sharpe_ratio=1.7, avg_gross_exposure_pct=14.0, avg_net_exposure_pct=14.0, days_in_market_pct=28.0, ) (run_dir / "metrics" / "metrics_summary.json").write_text(metrics.model_dump_json()) _write_run("train", 40.0) _write_run("valid", 20.0) _write_run("test", 25.0) wfv_dir = runs_dir / f"{experiment_name}_wfv" / "walk_forward" wfv_dir.mkdir(parents=True) wfv_summary = WalkForwardSummary( train_days=504, test_days=63, step_days=63, fold_count=2, folds=[ WalkForwardFoldResult( fold_index=0, train_start="2024-01-01", train_end="2025-12-31", test_start="2026-01-01", test_end="2026-03-31", train_metrics=SplitResult(run_id="wf_train_0", trade_count=20, total_return_pct=40.0), test_metrics=SplitResult( run_id="wf_test_0", trade_count=12, total_return_pct=8.0, win_rate=0.58, profit_factor=1.8, max_drawdown_pct=3.0, ), train_run_id="wf_train_0", test_run_id="wf_test_0", ), WalkForwardFoldResult( fold_index=1, train_start="2024-03-01", train_end="2026-02-28", test_start="2026-03-01", test_end="2026-05-31", train_metrics=SplitResult(run_id="wf_train_1", trade_count=20, total_return_pct=42.0), test_metrics=SplitResult( run_id="wf_test_1", trade_count=11, total_return_pct=6.0, win_rate=0.55, profit_factor=1.6, max_drawdown_pct=2.5, ), train_run_id="wf_train_1", test_run_id="wf_test_1", ), ], train_aggregate=WalkForwardAggregate(mean_return_pct=41.0), test_aggregate=WalkForwardAggregate( mean_return_pct=7.0, median_return_pct=7.0, worst_return_pct=6.0, positive_fold_rate_pct=100.0, mean_profit_factor=1.7, mean_max_drawdown_pct=2.75, mean_trade_count=11.5, mean_win_rate=0.565, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=34.0, worst_train_test_return_gap_pct=36.0, fold_return_cv=0.4, ), ) (wfv_dir / "walk_forward_summary.json").write_text(wfv_summary.model_dump_json()) rm_dir = runs_dir / f"{experiment_name}_rm" / "robustness_matrix" rm_dir.mkdir(parents=True) rm_summary = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=80.0, overall_worst_return_pct=-2.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, mean_return_pct=4.5, median_return_pct=3.5, worst_return_pct=-1.0, positive_window_rate_pct=83.3, mean_max_drawdown_pct=3.2, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=21.0, median_return_pct=18.0, worst_return_pct=8.0, positive_window_rate_pct=100.0, mean_max_drawdown_pct=7.0, ), ], ) (rm_dir / "robustness_matrix_summary.json").write_text(rm_summary.model_dump_json()) oot_dir = runs_dir / f"{experiment_name}_oot_rm" / "robustness_matrix" oot_dir.mkdir(parents=True) oot_summary = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=8, overall_positive_window_rate_pct=75.0, overall_worst_return_pct=-3.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=5, mean_return_pct=3.5, median_return_pct=2.5, worst_return_pct=-1.5, positive_window_rate_pct=80.0, mean_max_drawdown_pct=3.5, ), RobustnessHorizonSummary( horizon_days=252, window_count=3, mean_return_pct=15.0, median_return_pct=14.0, worst_return_pct=5.0, positive_window_rate_pct=100.0, mean_max_drawdown_pct=7.5, ), ], ) (oot_dir / "robustness_matrix_summary.json").write_text(oot_summary.model_dump_json()) synced = sync_official_manifests(journal_path, runs_dir, configs_dir) assert [entry.experiment_name for entry in synced] == [experiment_name] persisted = load_journal(journal_path) assert [entry.experiment_name for entry in persisted] == [experiment_name] assert persisted[0].walk_forward_summary is not None assert persisted[0].robustness_matrix_summary is not None assert persisted[0].out_of_time_robustness_summary is not None assert persisted[0].verdict_reasoning == "Auto-synced from official manifest." def test_sync_official_manifests_skips_missing_public_validation(self, tmp_path): journal_path = tmp_path / "journal.jsonl" configs_dir = tmp_path / "configs" / "experiments" runs_dir = tmp_path / "runs" configs_dir.mkdir(parents=True) experiment_name = "return_max_long_v8.32" (configs_dir / f"{experiment_name}.json").write_text( json.dumps( { "experiment_name": experiment_name, "description": "tier3 candidate without oot", } ) ) for idx, split in enumerate(("train", "valid", "test"), start=1): run_dir = runs_dir / "batch" / f"bt_{split}" (run_dir / "metrics").mkdir(parents=True) (run_dir / "manifest.json").write_text(json.dumps({"experiment_name": experiment_name})) (run_dir / "metadata.json").write_text( json.dumps( { "run_id": f"bt_{split}", "split_name": split, "started_at": f"2026-03-21T0{idx}:00:00+00:00", } ) ) metrics = MetricsBundle( trade_count=15, profit_factor=2.0, total_return_pct=20.0 + idx, win_rate=0.6, monthly_win_rate=0.75, equity_curve_r_squared=0.5, max_drawdown_pct=3.0, sharpe_ratio=1.7, avg_gross_exposure_pct=14.0, avg_net_exposure_pct=14.0, days_in_market_pct=28.0, ) (run_dir / "metrics" / "metrics_summary.json").write_text(metrics.model_dump_json()) wfv_dir = runs_dir / f"{experiment_name}_wfv" / "walk_forward" wfv_dir.mkdir(parents=True) (wfv_dir / "walk_forward_summary.json").write_text( WalkForwardSummary( train_days=504, test_days=63, step_days=63, fold_count=1, folds=[], train_aggregate=WalkForwardAggregate(mean_return_pct=35.0), test_aggregate=WalkForwardAggregate( mean_return_pct=7.0, median_return_pct=7.0, worst_return_pct=7.0, positive_fold_rate_pct=100.0, mean_profit_factor=1.7, mean_max_drawdown_pct=2.75, mean_trade_count=11.5, mean_win_rate=0.565, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=28.0, worst_train_test_return_gap_pct=28.0, fold_return_cv=0.0, ), ).model_dump_json() ) rm_dir = runs_dir / f"{experiment_name}_rm" / "robustness_matrix" rm_dir.mkdir(parents=True) (rm_dir / "robustness_matrix_summary.json").write_text( RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=80.0, overall_worst_return_pct=-2.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, mean_return_pct=4.5, median_return_pct=3.5, worst_return_pct=-1.0, positive_window_rate_pct=83.3, mean_max_drawdown_pct=3.2, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=21.0, median_return_pct=18.0, worst_return_pct=8.0, positive_window_rate_pct=100.0, mean_max_drawdown_pct=7.0, ), ], ).model_dump_json() ) synced = sync_official_manifests(journal_path, runs_dir, configs_dir) assert synced == [] assert load_journal(journal_path) == [] def test_sync_official_manifests_ignores_common_window_snapshot_override_runs(self, tmp_path): journal_path = tmp_path / "journal.jsonl" configs_dir = tmp_path / "configs" / "experiments" runs_dir = tmp_path / "runs" configs_dir.mkdir(parents=True) experiment_name = "return_max_long_v8.32" manifest_payload = { "experiment_name": experiment_name, "description": "tier3 candidate", "dataset_snapshot_id": "midlarge-liquid-long-v1_bucketfix_full_audit_tier3", } (configs_dir / f"{experiment_name}.json").write_text(json.dumps(manifest_payload)) def _write_run(split: str, run_name: str, total_return_pct: float, snapshot_id: str, started_at: str) -> None: run_dir = runs_dir / "batch" / run_name (run_dir / "metrics").mkdir(parents=True) (run_dir / "manifest.json").write_text(json.dumps(manifest_payload)) (run_dir / "resolved_config.json").write_text(json.dumps({"dataset_snapshot_id": snapshot_id})) (run_dir / "metadata.json").write_text( json.dumps( { "run_id": run_name, "split_name": split, "started_at": started_at, } ) ) metrics = MetricsBundle( trade_count=15, profit_factor=2.0, total_return_pct=total_return_pct, win_rate=0.6, monthly_win_rate=0.75, equity_curve_r_squared=0.5, max_drawdown_pct=3.0, sharpe_ratio=1.7, avg_gross_exposure_pct=14.0, avg_net_exposure_pct=14.0, days_in_market_pct=28.0, ) (run_dir / "metrics" / "metrics_summary.json").write_text(metrics.model_dump_json()) _write_run("train", "bt_train", 40.0, "midlarge-liquid-long-v1_bucketfix_full_audit_tier3", "2026-03-21T01:00:00+00:00") _write_run("valid", "bt_valid", 20.0, "midlarge-liquid-long-v1_bucketfix_full_audit_tier3", "2026-03-21T02:00:00+00:00") _write_run("test", "bt_test", 25.0, "midlarge-liquid-long-v1_bucketfix_full_audit_tier3", "2026-03-21T03:00:00+00:00") _write_run("train", "bt_common_train", 250.0, "midlarge-liquid-long-v1_bucketfix_full_audit_tier3_merged", "2026-03-22T01:00:00+00:00") wfv_dir = runs_dir / f"{experiment_name}_wfv" / "walk_forward" wfv_dir.mkdir(parents=True) wfv_summary = WalkForwardSummary( train_days=504, test_days=63, step_days=63, fold_count=1, folds=[], train_aggregate=WalkForwardAggregate(mean_return_pct=40.0), test_aggregate=WalkForwardAggregate( mean_return_pct=7.0, median_return_pct=7.0, worst_return_pct=7.0, positive_fold_rate_pct=100.0, mean_profit_factor=1.7, mean_max_drawdown_pct=2.75, mean_trade_count=11.5, mean_win_rate=0.565, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=33.0, worst_train_test_return_gap_pct=33.0, fold_return_cv=0.0, ), ) (wfv_dir / "walk_forward_summary.json").write_text(wfv_summary.model_dump_json()) rm_dir = runs_dir / f"{experiment_name}_rm" / "robustness_matrix" rm_dir.mkdir(parents=True) rm_summary = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=80.0, overall_worst_return_pct=-2.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, mean_return_pct=4.5, median_return_pct=3.5, worst_return_pct=-1.0, positive_window_rate_pct=83.3, mean_max_drawdown_pct=3.2, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=21.0, median_return_pct=18.0, worst_return_pct=8.0, positive_window_rate_pct=100.0, mean_max_drawdown_pct=7.0, ), ], ) (rm_dir / "robustness_matrix_summary.json").write_text(rm_summary.model_dump_json()) oot_dir = runs_dir / f"{experiment_name}_oot_rm" / "robustness_matrix" oot_dir.mkdir(parents=True) oot_summary = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=8, overall_positive_window_rate_pct=75.0, overall_worst_return_pct=-3.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=5, mean_return_pct=3.5, median_return_pct=2.5, worst_return_pct=-1.5, positive_window_rate_pct=80.0, mean_max_drawdown_pct=3.5, ), RobustnessHorizonSummary( horizon_days=252, window_count=3, mean_return_pct=15.0, median_return_pct=14.0, worst_return_pct=5.0, positive_window_rate_pct=100.0, mean_max_drawdown_pct=7.5, ), ], ) (oot_dir / "robustness_matrix_summary.json").write_text(oot_summary.model_dump_json()) synced = sync_official_manifests(journal_path, runs_dir, configs_dir) assert [entry.experiment_name for entry in synced] == [experiment_name] persisted = load_journal(journal_path) assert persisted[0].results["train"].run_id == "bt_train" assert persisted[0].results["train"].total_return_pct == 40.0 def test_sync_official_manifests_ignores_hidden_cache_json(self, tmp_path): journal_path = tmp_path / "journal.jsonl" configs_dir = tmp_path / "configs" / "experiments" runs_dir = tmp_path / "runs" configs_dir.mkdir(parents=True) experiment_name = "return_max_long_v10.95" (configs_dir / ".index.json").write_text('{"cached": true}{"stale": true}') (configs_dir / f"{experiment_name}.json").write_text( json.dumps( { "experiment_name": experiment_name, "description": "candidate", } ) ) for idx, split in enumerate(("train", "valid", "test"), start=1): run_dir = runs_dir / "batch" / f"bt_{split}" (run_dir / "metrics").mkdir(parents=True) (run_dir / "manifest.json").write_text(json.dumps({"experiment_name": experiment_name})) (run_dir / "metadata.json").write_text( json.dumps( { "run_id": f"bt_{split}", "split_name": split, "started_at": f"2026-03-2{idx}T01:00:00+00:00", } ) ) metrics = MetricsBundle( trade_count=20, profit_factor=1.8, total_return_pct=20.0 + idx, win_rate=0.58, monthly_win_rate=0.65, equity_curve_r_squared=0.45, max_drawdown_pct=3.0, sharpe_ratio=1.5, avg_gross_exposure_pct=14.0, avg_net_exposure_pct=14.0, days_in_market_pct=28.0, ) (run_dir / "metrics" / "metrics_summary.json").write_text(metrics.model_dump_json()) wfv_dir = runs_dir / f"{experiment_name}_wfv" / "walk_forward" wfv_dir.mkdir(parents=True) (wfv_dir / "walk_forward_summary.json").write_text( WalkForwardSummary( train_days=504, test_days=63, step_days=63, fold_count=1, folds=[], train_aggregate=WalkForwardAggregate(mean_return_pct=35.0), test_aggregate=WalkForwardAggregate( mean_return_pct=7.0, median_return_pct=7.0, worst_return_pct=7.0, positive_fold_rate_pct=100.0, mean_profit_factor=1.7, mean_max_drawdown_pct=2.75, mean_trade_count=11.5, mean_win_rate=0.565, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=28.0, worst_train_test_return_gap_pct=28.0, fold_return_cv=0.0, ), ).model_dump_json() ) rm_dir = runs_dir / f"{experiment_name}_rm" / "robustness_matrix" rm_dir.mkdir(parents=True) rm_summary = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=80.0, overall_worst_return_pct=-2.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, mean_return_pct=4.5, median_return_pct=3.5, worst_return_pct=-1.0, positive_window_rate_pct=83.3, mean_max_drawdown_pct=3.2, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=21.0, median_return_pct=18.0, worst_return_pct=8.0, positive_window_rate_pct=100.0, mean_max_drawdown_pct=7.0, ), ], ) (rm_dir / "robustness_matrix_summary.json").write_text(rm_summary.model_dump_json()) oot_dir = runs_dir / f"{experiment_name}_oot_rm" / "robustness_matrix" oot_dir.mkdir(parents=True) (oot_dir / "robustness_matrix_summary.json").write_text(rm_summary.model_dump_json()) synced = sync_official_manifests(journal_path, runs_dir, configs_dir) assert [entry.experiment_name for entry in synced] == [experiment_name] def test_leveraged_return_max_entries_are_hidden_by_default(self, tmp_path): journal_path = tmp_path / "journal.jsonl" registry_path = tmp_path / "registry.json" leaderboard_path = tmp_path / "LEADERBOARD.md" wfv, robustness, oot = self._validated_summary_triplet() valid_result = SplitResult( run_id="bt_valid", trade_count=5, profit_factor=1.6, total_return_pct=1.8, win_rate=0.4, monthly_win_rate=0.33, equity_curve_r_squared=0.02, avg_gross_exposure_pct=4.7, avg_net_exposure_pct=4.7, days_in_market_pct=10.8, ) test_result = SplitResult( run_id="bt_test", trade_count=13, profit_factor=3.8, total_return_pct=5.4, win_rate=0.54, monthly_win_rate=0.6, equity_curve_r_squared=0.63, avg_gross_exposure_pct=7.3, avg_net_exposure_pct=7.3, days_in_market_pct=32.6, ) append_journal_entry( journal_path, JournalEntry( entry_id="IMP-0001", timestamp="2026-03-18T16:35:14+00:00", experiment_name="return_max_long_v47_same_day_longer_tail_size200_bp150", hypothesis="leveraged", sqs_score=13.3, results={"valid": valid_result, "test": test_result}, walk_forward_summary=wfv, robustness_matrix_summary=robustness, out_of_time_robustness_summary=oot, ), ) append_journal_entry( journal_path, JournalEntry( entry_id="IMP-0002", timestamp="2026-03-18T16:40:14+00:00", experiment_name="return_max_long_v42_same_day_longer_tail_size200", hypothesis="unlevered", sqs_score=13.3, results={"valid": valid_result, "test": test_result}, walk_forward_summary=wfv, robustness_matrix_summary=robustness, out_of_time_robustness_summary=oot, ), ) registry = rebuild_registry(journal_path, registry_path, leaderboard_path) leveraged_entry = next(entry for entry in registry.entries if "_bp" in entry.experiment_name) assert leveraged_entry.strategy_family == "leveraged_return_max_long" assert leveraged_entry.is_retired is True visible = filter_registry_entries(registry.entries, include_retired=False) assert [entry.experiment_name for entry in visible] == [ "return_max_long_v42_same_day_longer_tail_size200" ] def test_incomplete_train_only_entries_are_hidden_by_default(self, tmp_path): journal_path = tmp_path / "journal.jsonl" registry_path = tmp_path / "registry.json" leaderboard_path = tmp_path / "LEADERBOARD.md" train_result = SplitResult( run_id="bt_train_only", trade_count=44, profit_factor=1.9, total_return_pct=33.5, win_rate=0.61, monthly_win_rate=0.53, equity_curve_r_squared=0.73, avg_gross_exposure_pct=18.7, avg_net_exposure_pct=18.7, days_in_market_pct=16.6, ) valid_result = SplitResult( run_id="bt_valid", trade_count=5, profit_factor=1.8, total_return_pct=1.1, win_rate=0.4, monthly_win_rate=0.33, equity_curve_r_squared=0.02, avg_gross_exposure_pct=4.7, avg_net_exposure_pct=4.7, days_in_market_pct=10.8, ) test_result = SplitResult( run_id="bt_test", trade_count=13, profit_factor=4.0, total_return_pct=5.4, win_rate=0.54, monthly_win_rate=0.6, equity_curve_r_squared=0.63, avg_gross_exposure_pct=7.3, avg_net_exposure_pct=7.3, days_in_market_pct=32.6, ) append_journal_entry( journal_path, JournalEntry( entry_id="IMP-0001", timestamp="2026-03-18T16:35:14+00:00", experiment_name="return_max_long_v48_same_day_longer_tail_size400_bp300", hypothesis="train ceiling", sqs_score=71.2, results={"train": train_result}, ), ) append_journal_entry( journal_path, JournalEntry( entry_id="IMP-0002", timestamp="2026-03-18T16:40:14+00:00", experiment_name="return_max_long_v47_same_day_longer_tail_size200_bp150", hypothesis="validated", sqs_score=13.3, results={"valid": valid_result, "test": test_result}, ), ) registry = rebuild_registry(journal_path, registry_path, leaderboard_path) visible = filter_registry_entries(registry.entries, include_retired=False) assert visible == [] lb_text = leaderboard_path.read_text() assert "return_max_long_v47_same_day_longer_tail_size200_bp150" not in lb_text assert "return_max_long_v48_same_day_longer_tail_size400_bp300" not in lb_text assert "incomplete train-only scans" in lb_text def test_pre_v6new29_entries_are_retired_by_default(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) (tmp_path / "configs" / "experiments").mkdir(parents=True) (tmp_path / "configs" / "experiments" / ".index.json").write_text( json.dumps( { "experiments": { "return_max_long_v6new.28": {"id": 28, "status": "retired"}, "return_max_long_v6new.29": {"id": 29, "status": "active"}, } }, indent=2, ) ) journal_path = tmp_path / "journal.jsonl" registry_path = tmp_path / "registry.json" leaderboard_path = tmp_path / "LEADERBOARD.md" train_result = SplitResult( run_id="bt_train", trade_count=20, profit_factor=2.1, total_return_pct=22.0, win_rate=0.6, monthly_win_rate=0.58, equity_curve_r_squared=0.61, avg_gross_exposure_pct=9.0, avg_net_exposure_pct=9.0, days_in_market_pct=28.0, ) valid_result = SplitResult( run_id="bt_valid", trade_count=7, profit_factor=3.0, total_return_pct=12.0, win_rate=0.57, monthly_win_rate=0.67, equity_curve_r_squared=0.71, avg_gross_exposure_pct=10.0, avg_net_exposure_pct=10.0, days_in_market_pct=30.0, ) test_result = SplitResult( run_id="bt_test", trade_count=8, profit_factor=3.2, total_return_pct=14.0, win_rate=0.62, monthly_win_rate=0.67, equity_curve_r_squared=0.72, avg_gross_exposure_pct=12.0, avg_net_exposure_pct=12.0, days_in_market_pct=35.0, ) wfv = WalkForwardSummary( window_mode="rolling_fixed", train_days=504, test_days=63, step_days=63, fold_count=8, folds=[], train_aggregate=WalkForwardAggregate( mean_return_pct=12.0, median_return_pct=10.0, worst_return_pct=1.0, positive_fold_rate_pct=100.0, mean_profit_factor=2.0, mean_max_drawdown_pct=4.0, mean_trade_count=10.0, mean_win_rate=0.6, ), test_aggregate=WalkForwardAggregate( mean_return_pct=8.0, median_return_pct=6.0, worst_return_pct=0.5, positive_fold_rate_pct=100.0, mean_profit_factor=2.0, mean_max_drawdown_pct=3.0, mean_trade_count=5.0, mean_win_rate=0.6, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=10.0, worst_train_test_return_gap_pct=20.0, fold_return_cv=0.5, ), ) robustness = RobustnessMatrixSummary( window_mode="rolling_horizon", horizons_days=[63, 252], step_days=21, overall_window_count=100, overall_positive_window_rate_pct=80.0, overall_worst_return_pct=-5.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=25, mean_return_pct=4.0, median_return_pct=3.5, worst_return_pct=-2.0, positive_window_rate_pct=80.0, mean_max_drawdown_pct=3.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=15, mean_return_pct=10.0, median_return_pct=9.0, worst_return_pct=-1.0, positive_window_rate_pct=90.0, mean_max_drawdown_pct=4.0, ), ], ) oot = RobustnessMatrixSummary( window_mode="rolling_horizon", horizons_days=[63, 252], step_days=21, overall_window_count=50, overall_positive_window_rate_pct=60.0, overall_worst_return_pct=-10.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=25, mean_return_pct=2.0, median_return_pct=1.0, worst_return_pct=-4.0, positive_window_rate_pct=60.0, mean_max_drawdown_pct=4.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=15, mean_return_pct=5.0, median_return_pct=4.0, worst_return_pct=-5.0, positive_window_rate_pct=70.0, mean_max_drawdown_pct=5.0, ), ], ) common = CommonWindowSummary( snapshot_id="cw", start_date=dt.date(2022, 3, 2), end_date=dt.date(2026, 3, 24), initial_equity=10_000.0, metrics=MetricsBundle( trade_count=50, profit_factor=3.0, total_return_pct=120.0, max_drawdown_pct=6.0, sharpe_ratio=2.0, avg_gross_exposure_pct=20.0, days_in_market_pct=50.0, ), ) append_journal_entry( journal_path, JournalEntry( entry_id="IMP-0605", timestamp="2026-03-25T15:20:00+00:00", experiment_name="return_max_long_v6new.28", hypothesis="pre cutoff", results={"train": train_result, "valid": valid_result, "test": test_result}, walk_forward_summary=wfv, robustness_matrix_summary=robustness, out_of_time_robustness_summary=oot, common_window_summary=common, ), ) append_journal_entry( journal_path, JournalEntry( entry_id="IMP-0606", timestamp="2026-03-25T15:28:32+00:00", experiment_name="return_max_long_v6new.29", hypothesis="cutoff", results={"train": train_result, "valid": valid_result, "test": test_result}, walk_forward_summary=wfv, robustness_matrix_summary=robustness, out_of_time_robustness_summary=oot, common_window_summary=common, ), ) registry = rebuild_registry(journal_path, registry_path, leaderboard_path) visible = filter_registry_entries(registry.entries, include_retired=False) assert [entry.experiment_name for entry in visible] == ["return_max_long_v6new.29"] retired = next( entry for entry in registry.entries if entry.experiment_name == "return_max_long_v6new.28" ) assert retired.is_retired is True visible_with_retired = filter_registry_entries(registry.entries, include_retired=True) assert {entry.experiment_name for entry in visible_with_retired} == { "return_max_long_v6new.29", "return_max_long_v6new.28", } lb_text = leaderboard_path.read_text() assert "return_max_long_v6new.29" in lb_text assert "return_max_long_v6new.28" not in lb_text assert "retired pre-IMP-0606 research" in lb_text def test_manifest_retired_entries_are_hidden_by_default(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) (tmp_path / "configs" / "experiments").mkdir(parents=True) (tmp_path / "configs" / "experiments" / ".index.json").write_text( json.dumps( { "experiments": { "return_max_long_v99_keep": {"id": 1, "status": "active"}, "return_max_long_v99_archive": {"id": 2, "status": "retired"}, } }, indent=2, ) ) journal_path = tmp_path / "journal.jsonl" registry_path = tmp_path / "registry.json" leaderboard_path = tmp_path / "LEADERBOARD.md" train_result = SplitResult(run_id="bt_train", trade_count=5, total_return_pct=12.0) valid_result = SplitResult(run_id="bt_valid", trade_count=5, total_return_pct=6.0) test_result = SplitResult(run_id="bt_test", trade_count=5, total_return_pct=7.0) append_journal_entry( journal_path, JournalEntry( entry_id="IMP-0900", timestamp="2026-03-31T12:00:00+00:00", experiment_name="return_max_long_v99_keep", hypothesis="keep", results={"train": train_result, "valid": valid_result, "test": test_result}, ), ) append_journal_entry( journal_path, JournalEntry( entry_id="IMP-0901", timestamp="2026-03-31T12:05:00+00:00", experiment_name="return_max_long_v99_archive", hypothesis="archive", results={"train": train_result, "valid": valid_result, "test": test_result}, ), ) registry = rebuild_registry(journal_path, registry_path, leaderboard_path) for entry in registry.entries: entry.sqs_score = 50.0 visible = filter_registry_entries(registry.entries, include_retired=False) assert [entry.experiment_name for entry in visible] == ["return_max_long_v99_keep"] visible_with_retired = filter_registry_entries(registry.entries, include_retired=True) assert {entry.experiment_name for entry in visible_with_retired} == { "return_max_long_v99_keep", "return_max_long_v99_archive", } lb_text = leaderboard_path.read_text() assert "return_max_long_v99_keep" in lb_text assert "return_max_long_v99_archive" not in lb_text # --------------------------------------------------------------------------- # WFQS v2: multiplicative penalty helpers # --------------------------------------------------------------------------- class TestGapPenalty: def test_low_gap_no_penalty(self): assert _gap_penalty(20.0) == 1.0 assert _gap_penalty(30.0) == 1.0 def test_moderate_gap(self): assert _gap_penalty(65.0) == pytest.approx(0.8, abs=0.01) def test_high_gap(self): assert _gap_penalty(100.0) == pytest.approx(0.6, abs=0.01) def test_very_high_gap(self): assert _gap_penalty(200.0) == pytest.approx(0.45, abs=0.01) def test_extreme_gap(self): assert _gap_penalty(300.0) == pytest.approx(0.3, abs=0.01) assert _gap_penalty(500.0) == 0.2 def test_none_gap(self): assert _gap_penalty(None) == 1.0 class TestOverfittingPenalty: """Tests for _overfitting_penalty — win-rate-based overfitting detection.""" def _summary(self, train_wr: float | None, test_wr: float | None, return_gap: float = 10.0) -> WalkForwardSummary: return WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=8, train_aggregate=WalkForwardAggregate(mean_win_rate=train_wr), test_aggregate=WalkForwardAggregate(mean_win_rate=test_wr), gap_stats=WalkForwardGapStats(mean_train_test_return_gap_pct=return_gap), ) def test_no_penalty_when_gap_below_3pp(self): """<=3pp win rate gap = statistical noise, no penalty.""" assert _overfitting_penalty(self._summary(0.68, 0.676)) == 1.0 # 0.4pp assert _overfitting_penalty(self._summary(0.70, 0.67)) == 1.0 # 3.0pp boundary def test_mild_penalty_between_3_and_10pp(self): """3-10pp: linear 1.00 → 0.85.""" # midpoint at 6.5pp → approx 0.925 result = _overfitting_penalty(self._summary(0.75, 0.685)) # 6.5pp assert 0.85 < result < 1.0 def test_significant_penalty_between_10_and_20pp(self): """10-20pp: linear 0.85 → 0.60.""" result = _overfitting_penalty(self._summary(0.75, 0.60)) # 15pp assert 0.60 < result < 0.85 def test_severe_penalty_above_20pp(self): """>=20pp: floor 0.50.""" assert _overfitting_penalty(self._summary(0.80, 0.55)) == 0.5 # 25pp assert _overfitting_penalty(self._summary(0.90, 0.40)) == 0.5 # 50pp def test_no_penalty_when_test_better_than_train(self): """Negative gap (test > train) = no overfitting signal, no penalty.""" assert _overfitting_penalty(self._summary(0.65, 0.70)) == 1.0 # test better assert _overfitting_penalty(self._summary(0.60, 0.80)) == 1.0 # test much better def test_fallback_to_gap_penalty_when_no_win_rate(self): """Falls back to _gap_penalty when aggregate win rates unavailable.""" # return gap of 523% → old _gap_penalty returns 0.2 summary = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=8, gap_stats=WalkForwardGapStats(mean_train_test_return_gap_pct=523.0), ) assert _overfitting_penalty(summary) == pytest.approx(0.2, abs=0.01) def test_fallback_when_only_test_wr_missing(self): """If test win rate is missing, fall back to old gap penalty.""" summary = self._summary(train_wr=0.70, test_wr=None, return_gap=20.0) # return gap 20% → _gap_penalty returns 1.0 assert _overfitting_penalty(summary) == 1.0 class TestFoldVariancePenalty: def test_low_cv_no_penalty(self): assert _fold_variance_penalty(0.3) == 1.0 assert _fold_variance_penalty(0.5) == 1.0 def test_moderate_cv(self): assert _fold_variance_penalty(1.0) == pytest.approx(0.85, abs=0.01) def test_high_cv(self): assert _fold_variance_penalty(1.5) == pytest.approx(0.7, abs=0.01) def test_extreme_cv(self): assert _fold_variance_penalty(2.0) == 0.6 def test_none_cv(self): assert _fold_variance_penalty(None) == 1.0 class TestTradeCredibility: def test_many_trades(self): assert _trade_credibility(25.0, 0.6) == 1.0 def test_few_trades(self): assert _trade_credibility(3.0, 0.6) == 0.5 def test_moderate_trades(self): assert _trade_credibility(7.0, 0.6) == 0.7 def test_suspicious_win_rate(self): assert _trade_credibility(12.0, 0.98) == pytest.approx(0.85 * 0.8, abs=0.01) def test_high_wr_with_enough_trades(self): # win_rate > 0.95 but mean_trades >= 15 → no additional penalty assert _trade_credibility(20.0, 0.98) == 1.0 def test_none_trades(self): assert _trade_credibility(None, 0.6) == 0.5 class TestEngineReliabilityPenalty: def test_high_ratio(self): assert _engine_reliability_penalty(0.8) == 1.0 def test_medium_ratio(self): assert _engine_reliability_penalty(0.5) == pytest.approx(0.75, abs=0.01) def test_low_ratio(self): assert _engine_reliability_penalty(0.2) == 0.4 def test_none_ratio(self): assert _engine_reliability_penalty(None) == 1.0 # --------------------------------------------------------------------------- # WFQS v2: full computation # --------------------------------------------------------------------------- class TestComputeWfqsV2: def test_returns_none_for_no_summary(self): score, breakdown = compute_wfqs_v2(None) assert score is None assert breakdown == {} def test_returns_none_for_zero_folds(self): summary = WalkForwardSummary(train_days=252, test_days=63, step_days=63, fold_count=0) score, breakdown = compute_wfqs_v2(summary) assert score is None def test_base_score_without_penalties(self): """When gap is low and CV is low, penalties are ~1.0 so wfqs_v2 ≈ base_score.""" summary = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=8, test_aggregate=WalkForwardAggregate( mean_return_pct=14.0, median_return_pct=12.0, worst_return_pct=2.0, positive_fold_rate_pct=87.5, mean_profit_factor=2.2, mean_max_drawdown_pct=4.0, mean_trade_count=25.0, mean_win_rate=0.6, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=18.0, worst_train_test_return_gap_pct=30.0, fold_return_cv=0.3, ), ) score, breakdown = compute_wfqs_v2(summary) assert score is not None assert score > 0 # All penalties should be ~1.0 assert breakdown["gap_penalty"] == 1.0 assert breakdown["fold_variance_penalty"] == 1.0 assert breakdown["trade_credibility"] == 1.0 assert breakdown["engine_reliability"] == 1.0 # base_score and final should be close assert abs(score - breakdown["base_score"]) < 1.0 def test_overfitted_profile_scores_lower(self): """v1045-like overfitted profile should score much lower than general-only-like.""" # General-only profile: low gap, low CV, good trades general = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=8, test_aggregate=WalkForwardAggregate( mean_return_pct=10.0, median_return_pct=8.0, worst_return_pct=1.0, positive_fold_rate_pct=87.5, mean_profit_factor=1.8, mean_max_drawdown_pct=5.0, mean_trade_count=30.0, mean_win_rate=0.55, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=37.0, worst_train_test_return_gap_pct=60.0, fold_return_cv=0.4, ), ) # Overfitted profile: extreme gap, high CV, few trades, high win rate overfit = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=8, test_aggregate=WalkForwardAggregate( mean_return_pct=20.0, median_return_pct=18.0, worst_return_pct=5.0, positive_fold_rate_pct=100.0, mean_profit_factor=3.0, mean_max_drawdown_pct=3.0, mean_trade_count=8.0, mean_win_rate=0.97, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=523.0, worst_train_test_return_gap_pct=900.0, fold_return_cv=1.8, ), ) general_score, _ = compute_wfqs_v2(general) overfit_score, overfit_breakdown = compute_wfqs_v2(overfit) assert general_score is not None assert overfit_score is not None # The overfit profile should be drastically penalized assert general_score > overfit_score # Gap penalty should be severe assert overfit_breakdown["gap_penalty"] == 0.2 # Fold variance should also penalize assert overfit_breakdown["fold_variance_penalty"] == 0.6 # Trade credibility should penalize (few trades + suspicious WR) assert overfit_breakdown["trade_credibility"] < 1.0 def test_recent_fold_strength_receives_modest_bonus(self): def _fold(i: int, end_date: dt.date, ret: float, pf: float = 1.8, dd: float = 5.0) -> WalkForwardFoldResult: metrics = SplitResult( run_id=f"bt_fold_{i}", trade_count=25, total_return_pct=ret, profit_factor=pf, max_drawdown_pct=dd, win_rate=0.55, ) return WalkForwardFoldResult( fold_index=i, train_start=end_date - dt.timedelta(days=315), train_end=end_date - dt.timedelta(days=64), test_start=end_date - dt.timedelta(days=63), test_end=end_date, train_run_id=f"train_{i}", test_run_id=f"test_{i}", train_metrics=metrics, test_metrics=metrics, ) common_kwargs = dict( train_days=252, test_days=63, step_days=63, fold_count=8, test_aggregate=WalkForwardAggregate( mean_return_pct=10.0, median_return_pct=8.0, worst_return_pct=1.0, positive_fold_rate_pct=87.5, mean_profit_factor=1.8, mean_max_drawdown_pct=5.0, mean_trade_count=25.0, mean_win_rate=0.55, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=20.0, fold_return_cv=0.3, ), ) strong_recent = WalkForwardSummary( **common_kwargs, folds=[ _fold(0, dt.date(2024, 2, 1), 6.0), _fold(1, dt.date(2024, 4, 1), 7.0), _fold(2, dt.date(2024, 6, 1), 8.0), _fold(3, dt.date(2024, 8, 1), 9.0), _fold(4, dt.date(2024, 10, 1), 14.0), _fold(5, dt.date(2024, 12, 1), 16.0), _fold(6, dt.date(2025, 2, 1), 18.0), _fold(7, dt.date(2025, 4, 1), 20.0), ], ) weak_recent = WalkForwardSummary( **common_kwargs, folds=[ _fold(0, dt.date(2024, 2, 1), 20.0), _fold(1, dt.date(2024, 4, 1), 18.0), _fold(2, dt.date(2024, 6, 1), 16.0), _fold(3, dt.date(2024, 8, 1), 14.0), _fold(4, dt.date(2024, 10, 1), 9.0), _fold(5, dt.date(2024, 12, 1), 8.0), _fold(6, dt.date(2025, 2, 1), 7.0), _fold(7, dt.date(2025, 4, 1), 6.0), ], ) strong_score, strong_breakdown = compute_wfqs_v2(strong_recent) weak_score, weak_breakdown = compute_wfqs_v2(weak_recent) assert strong_score is not None assert weak_score is not None assert strong_breakdown["recent_fold_count"] >= 2 assert strong_breakdown["recent_quality"] > weak_breakdown["recent_quality"] assert strong_score > weak_score def test_engine_reliability_applied(self): """Engine reliability penalty should reduce score when ratio is low.""" summary = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=8, test_aggregate=WalkForwardAggregate( mean_return_pct=10.0, median_return_pct=8.0, worst_return_pct=1.0, positive_fold_rate_pct=87.5, mean_profit_factor=1.8, mean_max_drawdown_pct=5.0, mean_trade_count=30.0, mean_win_rate=0.55, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=20.0, fold_return_cv=0.3, ), engine_reliability_ratio=0.2, ) score, breakdown = compute_wfqs_v2(summary) assert score is not None assert breakdown["engine_reliability"] == 0.4 # Without engine reliability summary_no_er = summary.model_copy(update={"engine_reliability_ratio": None}) score_no_er, _ = compute_wfqs_v2(summary_no_er) assert score_no_er is not None assert score_no_er > score def test_single_fold(self): summary = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=1, test_aggregate=WalkForwardAggregate( mean_return_pct=5.0, median_return_pct=5.0, worst_return_pct=5.0, positive_fold_rate_pct=100.0, mean_profit_factor=1.5, mean_max_drawdown_pct=3.0, mean_trade_count=10.0, mean_win_rate=0.5, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=10.0, ), ) score, breakdown = compute_wfqs_v2(summary) assert score is not None # Low fold penalty applied assert breakdown["base_score"] > score / breakdown["gap_penalty"] # base was penalized def test_zero_trades(self): summary = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=4, test_aggregate=WalkForwardAggregate( mean_return_pct=0.0, median_return_pct=0.0, worst_return_pct=0.0, positive_fold_rate_pct=0.0, mean_trade_count=0.0, mean_win_rate=0.0, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=0.0, ), ) score, breakdown = compute_wfqs_v2(summary) assert score is not None assert breakdown["trade_credibility"] == 0.5 class TestComputePublicSqsV2: def test_returns_none_without_rqs(self): score, breakdown, source = compute_public_sqs_v2(None, None, None) assert score is None assert source is None def test_requires_validations(self): train = SplitResult(run_id="bt_train", trade_count=10, total_return_pct=20.0, win_rate=0.7) valid = SplitResult(run_id="bt_valid", trade_count=10, total_return_pct=10.0, win_rate=0.6) test = SplitResult(run_id="bt_test", trade_count=10, total_return_pct=12.0, win_rate=0.6) score, breakdown, source = compute_public_sqs_v2(train, valid, test) assert score is None assert source == "pending_validation" assert breakdown["requires_walk_forward"] == 1.0 assert breakdown["requires_robustness"] == 1.0 assert breakdown["requires_out_of_time_robustness"] == 1.0 def test_deployment_with_wfqs_v2(self): train = SplitResult(run_id="bt_train", trade_count=10, total_return_pct=20.0, win_rate=0.7) valid = SplitResult(run_id="bt_valid", trade_count=10, total_return_pct=10.0, win_rate=0.6) test = SplitResult(run_id="bt_test", trade_count=10, total_return_pct=12.0, win_rate=0.6) summary = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=8, test_aggregate=WalkForwardAggregate( mean_return_pct=10.0, median_return_pct=8.0, worst_return_pct=1.0, positive_fold_rate_pct=87.5, mean_profit_factor=1.8, mean_max_drawdown_pct=5.0, mean_trade_count=25.0, mean_win_rate=0.55, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=20.0, fold_return_cv=0.3, ), ) robustness = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=90.0, overall_worst_return_pct=-2.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, mean_return_pct=4.0, median_return_pct=3.5, worst_return_pct=-1.0, positive_window_rate_pct=83.3, mean_max_drawdown_pct=3.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=18.0, median_return_pct=12.0, worst_return_pct=4.0, positive_window_rate_pct=100.0, mean_max_drawdown_pct=5.0, ), ], ) oot = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=80.0, overall_worst_return_pct=-4.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, mean_return_pct=3.0, median_return_pct=3.0, worst_return_pct=-1.5, positive_window_rate_pct=66.7, mean_max_drawdown_pct=3.2, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=11.0, median_return_pct=9.0, worst_return_pct=2.0, positive_window_rate_pct=75.0, mean_max_drawdown_pct=5.2, ), ], ) score, breakdown, source = compute_public_sqs_v2( train, valid, test, walk_forward_summary=summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=oot, ) assert score is not None assert source == "v2_deployment+robustness+oot" assert breakdown["oot_gate_factor"] > 0.0 def test_out_of_time_robustness_applies_additional_gate(self): train = SplitResult(run_id="bt_train", trade_count=10, total_return_pct=20.0, win_rate=0.7) valid = SplitResult(run_id="bt_valid", trade_count=10, total_return_pct=10.0, win_rate=0.6) test = SplitResult(run_id="bt_test", trade_count=10, total_return_pct=12.0, win_rate=0.6) summary = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=8, test_aggregate=WalkForwardAggregate( mean_return_pct=10.0, median_return_pct=8.0, worst_return_pct=1.0, positive_fold_rate_pct=87.5, mean_profit_factor=1.8, mean_max_drawdown_pct=5.0, mean_trade_count=25.0, mean_win_rate=0.55, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=20.0, fold_return_cv=0.3, ), ) robustness = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=90.0, overall_worst_return_pct=-2.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, mean_return_pct=4.0, median_return_pct=3.5, worst_return_pct=-1.0, positive_window_rate_pct=83.3, mean_max_drawdown_pct=3.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=18.0, median_return_pct=12.0, worst_return_pct=4.0, positive_window_rate_pct=100.0, mean_max_drawdown_pct=5.0, ), ], ) oot = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=60.0, overall_worst_return_pct=-15.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, mean_return_pct=2.0, median_return_pct=2.0, worst_return_pct=-2.0, positive_window_rate_pct=66.7, mean_max_drawdown_pct=3.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=7.0, median_return_pct=7.0, worst_return_pct=-1.0, positive_window_rate_pct=75.0, mean_max_drawdown_pct=5.0, ), ], ) strong_oot = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=85.0, overall_worst_return_pct=-3.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, mean_return_pct=3.0, median_return_pct=3.0, worst_return_pct=-1.0, positive_window_rate_pct=83.3, mean_max_drawdown_pct=2.5, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=10.0, median_return_pct=8.5, worst_return_pct=1.5, positive_window_rate_pct=100.0, mean_max_drawdown_pct=4.8, ), ], ) strong_score, _, _ = compute_public_sqs_v2( train, valid, test, walk_forward_summary=summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=strong_oot, ) oot_score, breakdown, source = compute_public_sqs_v2( train, valid, test, walk_forward_summary=summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=oot, ) assert strong_score is not None assert oot_score is not None assert oot_score < strong_score assert source == "v2_deployment+robustness+oot" assert "oot_gate_factor" in breakdown assert "oot_quality_factor" in breakdown assert "oot_quality" in breakdown def test_low_activity_strategy_gets_lower_public_score(self): train = SplitResult(run_id="bt_train", trade_count=20, total_return_pct=20.0, win_rate=0.7) valid = SplitResult(run_id="bt_valid", trade_count=15, total_return_pct=10.0, win_rate=0.6) active_test = SplitResult( run_id="bt_test_active", trade_count=24, total_return_pct=12.0, win_rate=0.6, days_in_market_pct=55.0, ) sparse_test = SplitResult( run_id="bt_test_sparse", trade_count=9, total_return_pct=12.0, win_rate=0.6, days_in_market_pct=18.0, ) active_summary = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=8, test_aggregate=WalkForwardAggregate( mean_return_pct=10.0, median_return_pct=8.0, worst_return_pct=1.0, positive_fold_rate_pct=87.5, mean_profit_factor=1.8, mean_max_drawdown_pct=5.0, mean_trade_count=12.0, mean_win_rate=0.55, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=20.0, fold_return_cv=0.3, ), ) sparse_summary = active_summary.model_copy( update={ "test_aggregate": active_summary.test_aggregate.model_copy( update={"mean_trade_count": 3.5} ) } ) robustness = RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=90.0, overall_worst_return_pct=-2.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, mean_return_pct=4.0, median_return_pct=3.5, worst_return_pct=-1.0, positive_window_rate_pct=83.3, mean_max_drawdown_pct=3.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, mean_return_pct=18.0, median_return_pct=12.0, worst_return_pct=4.0, positive_window_rate_pct=100.0, mean_max_drawdown_pct=5.0, ), ], ) active_score, active_breakdown, _ = compute_public_sqs_v2( train, valid, active_test, walk_forward_summary=active_summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=robustness, ) sparse_score, sparse_breakdown, _ = compute_public_sqs_v2( train, valid, sparse_test, walk_forward_summary=sparse_summary, robustness_matrix_summary=robustness, out_of_time_robustness_summary=robustness, ) assert active_score is not None assert sparse_score is not None assert sparse_score < active_score assert active_breakdown["activity_factor"] > sparse_breakdown["activity_factor"] # --------------------------------------------------------------------------- # SQS v9: 3-pillar additive core + scenario regime score # --------------------------------------------------------------------------- def _make_good_wfv() -> WalkForwardSummary: return WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=8, test_aggregate=WalkForwardAggregate( mean_return_pct=12.0, median_return_pct=10.0, worst_return_pct=1.0, positive_fold_rate_pct=87.5, mean_profit_factor=2.0, mean_max_drawdown_pct=5.0, mean_trade_count=15.0, mean_win_rate=0.65, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=15.0, fold_return_cv=0.25, ), ) def _make_good_robustness() -> RobustnessMatrixSummary: return RobustnessMatrixSummary( horizons_days=[63, 252], step_days=21, overall_window_count=10, overall_positive_window_rate_pct=90.0, overall_worst_return_pct=-1.0, horizon_summaries=[ RobustnessHorizonSummary( horizon_days=63, window_count=6, positive_window_rate_pct=90.0, mean_return_pct=8.0, median_return_pct=7.0, worst_return_pct=-1.0, mean_max_drawdown_pct=4.0, ), RobustnessHorizonSummary( horizon_days=252, window_count=4, positive_window_rate_pct=100.0, mean_return_pct=12.0, median_return_pct=12.0, worst_return_pct=4.0, mean_max_drawdown_pct=5.0, ), ], ) class TestSQSv9: def test_basic_3_pillar_score(self): """v9 core = RQS×0.35 + WFQS_v2×0.40 + regime×0.25.""" train = SplitResult(run_id="bt_train", trade_count=30, total_return_pct=25.0, win_rate=0.75) valid = SplitResult(run_id="bt_valid", trade_count=20, total_return_pct=12.0, win_rate=0.65) test = SplitResult(run_id="bt_test", trade_count=25, total_return_pct=15.0, win_rate=0.65, days_in_market_pct=50.0) wfv = _make_good_wfv() rob = _make_good_robustness() rrs = 75.0 score, breakdown, source = compute_public_sqs_v9( train, valid, test, walk_forward_summary=wfv, robustness_matrix_summary=rob, scenario_robustness_score=rrs, ) assert score is not None assert breakdown["regime_score"] == rrs assert breakdown["core_score"] == round( breakdown["rqs_score"] * 0.35 + breakdown["wfqs_v2_score"] * 0.40 + rrs * 0.25, 1 ) assert "scenario_rrs" in (source or "") def test_regime_fallback_chain_uses_scenario_first(self): """scenario_robustness_score takes priority over OOT.""" score_scenario, regime_source_scenario = _resolve_regime_score(80.0, None) assert score_scenario == 80.0 assert regime_source_scenario == "scenario_rrs" def test_regime_fallback_chain_uses_oot_when_no_scenario(self): """Falls back to OOT quality when no scenario score is provided.""" rob = _make_good_robustness() score, source = _resolve_regime_score(None, rob) assert score is not None assert source == "oot_quality_fallback" def test_regime_fallback_chain_neutral_when_nothing(self): """Falls back to 50.0 neutral when neither scenario nor OOT is available.""" score, source = _resolve_regime_score(None, None) assert score == 50.0 assert source == "neutral_fallback" def test_pending_when_no_wfv(self): """Returns pending_validation when walk-forward is missing.""" train = SplitResult(run_id="bt_train", trade_count=20, total_return_pct=20.0) valid = SplitResult(run_id="bt_valid", trade_count=15, total_return_pct=10.0) test = SplitResult(run_id="bt_test", trade_count=18, total_return_pct=12.0) score, breakdown, source = compute_public_sqs_v9( train, valid, test, scenario_robustness_score=70.0 ) assert score is None assert source == "pending_validation" def test_pending_when_no_robustness(self): """Returns pending_validation when robustness matrix is missing.""" train = SplitResult(run_id="bt_train", trade_count=20, total_return_pct=20.0) valid = SplitResult(run_id="bt_valid", trade_count=15, total_return_pct=10.0) test = SplitResult(run_id="bt_test", trade_count=18, total_return_pct=12.0) score, breakdown, source = compute_public_sqs_v9( train, valid, test, walk_forward_summary=_make_good_wfv(), scenario_robustness_score=70.0, ) assert score is None assert source == "pending_validation" def test_pending_when_no_oot_and_no_scenario(self): """Returns pending_validation when OOT and scenario_robustness_score are both absent.""" train = SplitResult(run_id="bt_train", trade_count=20, total_return_pct=20.0) valid = SplitResult(run_id="bt_valid", trade_count=15, total_return_pct=10.0) test = SplitResult(run_id="bt_test", trade_count=18, total_return_pct=12.0) score, breakdown, source = compute_public_sqs_v9( train, valid, test, walk_forward_summary=_make_good_wfv(), robustness_matrix_summary=_make_good_robustness(), ) assert score is None assert source == "pending_validation" assert breakdown.get("requires_out_of_time_robustness") == 1.0 def test_deployment_gate_reduces_score(self): """Poor WFV gate (0 criteria passed) produces score multiplied by 0.40.""" train = SplitResult(run_id="bt_train", trade_count=30, total_return_pct=25.0, win_rate=0.75) valid = SplitResult(run_id="bt_valid", trade_count=20, total_return_pct=12.0, win_rate=0.65) test = SplitResult(run_id="bt_test", trade_count=25, total_return_pct=15.0, win_rate=0.65, days_in_market_pct=50.0) good_wfv = _make_good_wfv() # Make a bad WFV that fails all 3 gate criteria bad_wfv = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=8, test_aggregate=WalkForwardAggregate( mean_return_pct=-5.0, median_return_pct=-8.0, worst_return_pct=-20.0, positive_fold_rate_pct=37.5, mean_profit_factor=0.8, mean_max_drawdown_pct=15.0, ), ) rob = _make_good_robustness() good_score, good_bd, _ = compute_public_sqs_v9( train, valid, test, walk_forward_summary=good_wfv, robustness_matrix_summary=rob, scenario_robustness_score=70.0, ) bad_score, bad_bd, _ = compute_public_sqs_v9( train, valid, test, walk_forward_summary=bad_wfv, robustness_matrix_summary=rob, scenario_robustness_score=70.0, ) assert good_score is not None assert bad_score is not None assert good_bd["deployment_gate_factor"] > bad_bd["deployment_gate_factor"] assert good_score > bad_score def test_scenario_overrides_oot_in_regime_score(self): """When scenario_robustness_score is set, OOT is ignored.""" rob = _make_good_robustness() # OOT would give some quality score from robustness data oot_quality, _ = compute_oot_robustness_quality(rob) # Scenario score 90 should win over OOT quality regardless score_with_scenario, regime_source = _resolve_regime_score(90.0, rob) assert score_with_scenario == 90.0 assert regime_source == "scenario_rrs" # Ensure OOT would give a different answer assert oot_quality != 90.0 or True # just confirming scenario wins def test_v9_higher_rrs_improves_score(self): """Higher scenario RRS yields higher v9 score (all else equal).""" train = SplitResult(run_id="bt_train", trade_count=30, total_return_pct=25.0, win_rate=0.75) valid = SplitResult(run_id="bt_valid", trade_count=20, total_return_pct=12.0, win_rate=0.65) test = SplitResult(run_id="bt_test", trade_count=25, total_return_pct=15.0, win_rate=0.65, days_in_market_pct=50.0) wfv = _make_good_wfv() rob = _make_good_robustness() score_low, _, _ = compute_public_sqs_v9( train, valid, test, walk_forward_summary=wfv, robustness_matrix_summary=rob, scenario_robustness_score=20.0, ) score_high, _, _ = compute_public_sqs_v9( train, valid, test, walk_forward_summary=wfv, robustness_matrix_summary=rob, scenario_robustness_score=90.0, ) assert score_low is not None and score_high is not None assert score_high > score_low