You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
2503 lines
97 KiB
Python
2503 lines
97 KiB
Python
"""Strategy improvement tracker: SQS computation, journal I/O, leaderboard."""
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import datetime as dt
|
|
import functools
|
|
import fcntl
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any, Callable, Iterator
|
|
|
|
from libs.backtest.domain import (
|
|
CommonWindowSummary,
|
|
ConfigDelta,
|
|
DeploymentScoreWeights,
|
|
ExperimentRegistry,
|
|
JournalEntry,
|
|
MetricsBundle,
|
|
RobustnessMatrixSummary,
|
|
WalkForwardScoreWeights,
|
|
WFQSv2Weights,
|
|
ReturnScoreWeights,
|
|
PromotionScoreWeights,
|
|
RegistryEntry,
|
|
SplitResult,
|
|
SQSWeights,
|
|
SQSv2Weights,
|
|
UnifiedScoreWeights,
|
|
WalkForwardAggregate,
|
|
WalkForwardSummary,
|
|
)
|
|
from libs.common.logging import get_logger
|
|
from libs.common.time_utils import utc_now
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
_DEFAULT_WEIGHTS = SQSWeights()
|
|
_DEFAULT_V2_WEIGHTS = SQSv2Weights()
|
|
_DEFAULT_PROMOTION_WEIGHTS = PromotionScoreWeights()
|
|
_DEFAULT_UNIFIED_WEIGHTS = UnifiedScoreWeights()
|
|
_DEFAULT_RQS_WEIGHTS = ReturnScoreWeights()
|
|
_DEFAULT_WFQS_WEIGHTS = WalkForwardScoreWeights()
|
|
_DEFAULT_WFQS_V2_WEIGHTS = WFQSv2Weights()
|
|
_DEFAULT_DEPLOYMENT_WEIGHTS = DeploymentScoreWeights()
|
|
_DEFAULT_PUBLIC_COMMON_WINDOW_WEIGHT = 0.20
|
|
_RETIRED_STRATEGY_FAMILIES = {
|
|
"short_core",
|
|
"legacy_pead",
|
|
"leveraged_return_max_long",
|
|
"exact_pocket_return_max_long",
|
|
"named_micro_return_max_long",
|
|
}
|
|
_EXPERIMENTS_DIR = Path("configs/experiments")
|
|
_NAMED_MICRO_ENGINE_TOKENS = (
|
|
"epam_micro",
|
|
"nrg_np12",
|
|
"glw_np11",
|
|
"pl_micro",
|
|
"exas_micro",
|
|
"aap_micro",
|
|
"apld_micro",
|
|
"czr_hot_micro",
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# SQS computation (pure functions, no I/O)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _normalize(value: float | None, low: float, high: float) -> float:
|
|
"""Linear normalise *value* to 0-100 between *low* (0 pts) and *high* (100 pts)."""
|
|
if value is None:
|
|
return 0.0
|
|
if high == low:
|
|
return 50.0
|
|
score = (value - low) / (high - low) * 100.0
|
|
return max(0.0, min(100.0, score))
|
|
|
|
|
|
def _normalize_inverse(value: float | None, low: float, high: float) -> float:
|
|
"""Like _normalize but lower values are better (e.g. drawdown)."""
|
|
if value is None:
|
|
return 0.0
|
|
if high == low:
|
|
return 50.0
|
|
# low = worst (0 pts), high = best (100 pts) — but for inverse metrics
|
|
# low value = good, high value = bad. Swap the interpretation.
|
|
score = (low - value) / (low - high) * 100.0
|
|
return max(0.0, min(100.0, score))
|
|
|
|
|
|
def _normalize_band(
|
|
value: float | None,
|
|
low_bad: float,
|
|
low_good: float,
|
|
high_good: float,
|
|
high_bad: float,
|
|
) -> float:
|
|
"""Score 0-100 with an optimal middle band."""
|
|
if value is None:
|
|
return 0.0
|
|
if value <= low_bad or value >= high_bad:
|
|
return 0.0
|
|
if low_good <= value <= high_good:
|
|
return 100.0
|
|
if value < low_good:
|
|
return (value - low_bad) / (low_good - low_bad) * 100.0
|
|
return (high_bad - value) / (high_bad - high_good) * 100.0
|
|
|
|
|
|
def _calibrate_sqs(value: float | None) -> float | None:
|
|
"""Compress raw integrated scores into a harsher absolute-looking range."""
|
|
if value is None:
|
|
return None
|
|
return round(max(0.0, value * 0.70 - 7.5), 1)
|
|
|
|
|
|
def _safe_ratio(numerator: float | None, denominator: float | None) -> float | None:
|
|
if numerator is None or denominator is None or denominator <= 0:
|
|
return None
|
|
return numerator / denominator
|
|
|
|
|
|
def _apply_single_split_penalty(value: float | None) -> float | None:
|
|
"""Discount scores that have no valid/test confirmation pair."""
|
|
if value is None:
|
|
return None
|
|
return round(value * 0.80, 1)
|
|
|
|
|
|
def compute_sqs(
|
|
metrics: MetricsBundle,
|
|
weights: SQSWeights | None = None,
|
|
) -> tuple[float, dict[str, float]]:
|
|
"""Compute Strategy Quality Score from test-split metrics.
|
|
|
|
Returns (sqs_score, breakdown_dict).
|
|
"""
|
|
w = weights or _DEFAULT_WEIGHTS
|
|
|
|
# --- Profitability (40%) ---
|
|
pf_score = _normalize(metrics.profit_factor, low=0.8, high=2.0)
|
|
ret_score = _normalize(metrics.total_return_pct, low=-5.0, high=5.0)
|
|
profitability = pf_score * 0.6 + ret_score * 0.4
|
|
|
|
# --- Risk (25%) ---
|
|
dd_score = _normalize_inverse(metrics.max_drawdown_pct, low=10.0, high=1.0)
|
|
sharpe_score = _normalize(metrics.sharpe_ratio, low=-1.0, high=2.0)
|
|
risk = dd_score * 0.5 + sharpe_score * 0.5
|
|
|
|
# --- Consistency (20%) ---
|
|
wr_score = _normalize(metrics.win_rate, low=0.35, high=0.65)
|
|
mwr_score = _normalize(metrics.monthly_win_rate, low=0.30, high=0.70)
|
|
consistency = wr_score * 0.5 + mwr_score * 0.5
|
|
|
|
# --- Robustness (15%) ---
|
|
r2_score = _normalize(metrics.equity_curve_r_squared, low=0.0, high=0.80)
|
|
tc_score = _normalize(float(metrics.trade_count), low=10.0, high=100.0)
|
|
robustness = r2_score * 0.5 + tc_score * 0.5
|
|
|
|
# Weighted total
|
|
sqs = (
|
|
profitability * w.profitability
|
|
+ risk * w.risk
|
|
+ consistency * w.consistency
|
|
+ robustness * w.robustness
|
|
)
|
|
|
|
# Low-trade penalty
|
|
if metrics.trade_count < w.low_trade_penalty_threshold:
|
|
sqs *= w.low_trade_penalty_factor
|
|
|
|
sqs = round(sqs, 1)
|
|
breakdown = {
|
|
"profitability": round(profitability, 1),
|
|
"risk": round(risk, 1),
|
|
"consistency": round(consistency, 1),
|
|
"robustness": round(robustness, 1),
|
|
}
|
|
return sqs, breakdown
|
|
|
|
|
|
def compute_sqs_v2(
|
|
metrics: MetricsBundle,
|
|
weights: SQSv2Weights | None = None,
|
|
) -> tuple[float | None, dict[str, float]]:
|
|
"""Compute SQS v2, adding a capital-efficiency sleeve to the score."""
|
|
if metrics.avg_gross_exposure_pct is None or metrics.days_in_market_pct is None:
|
|
return None, {}
|
|
|
|
w = weights or _DEFAULT_V2_WEIGHTS
|
|
|
|
pf_score = _normalize(metrics.profit_factor, low=0.8, high=2.0)
|
|
ret_score = _normalize(metrics.total_return_pct, low=-5.0, high=5.0)
|
|
profitability = pf_score * 0.6 + ret_score * 0.4
|
|
|
|
dd_score = _normalize_inverse(metrics.max_drawdown_pct, low=10.0, high=1.0)
|
|
sharpe_score = _normalize(metrics.sharpe_ratio, low=-1.0, high=2.0)
|
|
risk = dd_score * 0.5 + sharpe_score * 0.5
|
|
|
|
wr_score = _normalize(metrics.win_rate, low=0.35, high=0.65)
|
|
mwr_score = _normalize(metrics.monthly_win_rate, low=0.30, high=0.70)
|
|
consistency = wr_score * 0.5 + mwr_score * 0.5
|
|
|
|
r2_score = _normalize(metrics.equity_curve_r_squared, low=0.0, high=0.80)
|
|
tc_score = _normalize(float(metrics.trade_count), low=10.0, high=100.0)
|
|
robustness = r2_score * 0.5 + tc_score * 0.5
|
|
|
|
return_on_gross_exposure = None
|
|
if metrics.total_return_pct is not None and metrics.avg_gross_exposure_pct > 0:
|
|
return_on_gross_exposure = metrics.total_return_pct / metrics.avg_gross_exposure_pct
|
|
roe_score = _normalize(return_on_gross_exposure, low=0.0, high=0.50)
|
|
dim_score = _normalize_band(
|
|
metrics.days_in_market_pct,
|
|
low_bad=10.0,
|
|
low_good=40.0,
|
|
high_good=80.0,
|
|
high_bad=100.0,
|
|
)
|
|
capital_efficiency = roe_score * 0.7 + dim_score * 0.3
|
|
|
|
sqs_v2 = (
|
|
profitability * w.profitability
|
|
+ risk * w.risk
|
|
+ consistency * w.consistency
|
|
+ robustness * w.robustness
|
|
+ capital_efficiency * w.capital_efficiency
|
|
)
|
|
|
|
if metrics.trade_count < w.low_trade_penalty_threshold:
|
|
sqs_v2 *= w.low_trade_penalty_factor
|
|
|
|
sqs_v2 = round(sqs_v2, 1)
|
|
breakdown = {
|
|
"profitability": round(profitability, 1),
|
|
"risk": round(risk, 1),
|
|
"consistency": round(consistency, 1),
|
|
"robustness": round(robustness, 1),
|
|
"capital_efficiency": round(capital_efficiency, 1),
|
|
}
|
|
return sqs_v2, breakdown
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helper builders
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def build_split_result(split_name: str, run_id: str, metrics: MetricsBundle) -> SplitResult:
|
|
"""Create a SplitResult from MetricsBundle."""
|
|
return SplitResult(
|
|
run_id=run_id,
|
|
trade_count=metrics.trade_count,
|
|
profit_factor=metrics.profit_factor,
|
|
total_return_pct=metrics.total_return_pct,
|
|
annualized_return_pct=metrics.annualized_return_pct,
|
|
win_rate=metrics.win_rate,
|
|
max_drawdown_pct=metrics.max_drawdown_pct,
|
|
sharpe_ratio=metrics.sharpe_ratio,
|
|
monthly_win_rate=metrics.monthly_win_rate,
|
|
equity_curve_r_squared=metrics.equity_curve_r_squared,
|
|
avg_gross_exposure_pct=metrics.avg_gross_exposure_pct,
|
|
avg_net_exposure_pct=metrics.avg_net_exposure_pct,
|
|
days_in_market_pct=metrics.days_in_market_pct,
|
|
)
|
|
|
|
|
|
def _metrics_from_split_result(result: SplitResult | None) -> MetricsBundle | None:
|
|
if result is None:
|
|
return None
|
|
payload = result.model_dump(exclude={"run_id"})
|
|
return MetricsBundle.model_validate(payload)
|
|
|
|
|
|
_metrics_path_index: dict[str, Path] | None = None
|
|
|
|
|
|
def _get_metrics_path_index() -> dict[str, Path]:
|
|
"""Build run_id -> metrics_summary.json path index (single rglob)."""
|
|
global _metrics_path_index
|
|
if _metrics_path_index is not None:
|
|
return _metrics_path_index
|
|
_metrics_path_index = {}
|
|
runs_root = Path("runs")
|
|
for metrics_path in runs_root.rglob("metrics/metrics_summary.json"):
|
|
# run_dir is metrics_path.parent.parent (e.g. runs/.../run_id/)
|
|
run_dir = metrics_path.parent.parent
|
|
run_dir_name = run_dir.name
|
|
# Index by directory name as run_id
|
|
if run_dir_name not in _metrics_path_index or metrics_path.stat().st_mtime > _metrics_path_index[run_dir_name].stat().st_mtime:
|
|
_metrics_path_index[run_dir_name] = metrics_path
|
|
return _metrics_path_index
|
|
|
|
|
|
@functools.lru_cache(maxsize=2048)
|
|
def _load_run_metrics_summary(run_id: str) -> dict[str, Any] | None:
|
|
runs_root = Path("runs")
|
|
metrics_path = runs_root / run_id / "metrics" / "metrics_summary.json"
|
|
if not metrics_path.exists():
|
|
idx = _get_metrics_path_index()
|
|
found = idx.get(run_id)
|
|
if not found:
|
|
return None
|
|
metrics_path = found
|
|
summary = json.loads(metrics_path.read_text())
|
|
needed_fields = (
|
|
"annualized_return_pct",
|
|
"avg_gross_exposure_pct",
|
|
"avg_net_exposure_pct",
|
|
"days_in_market_pct",
|
|
)
|
|
if all(summary.get(field) is not None for field in needed_fields):
|
|
return summary
|
|
|
|
equity_curve_path = metrics_path.parent.parent / "artifacts" / "daily_equity_curve.parquet"
|
|
if not equity_curve_path.exists():
|
|
return summary
|
|
|
|
try:
|
|
import pyarrow.parquet as pq
|
|
|
|
rows = pq.read_table(
|
|
equity_curve_path,
|
|
columns=["equity", "gross_exposure", "net_exposure"],
|
|
).to_pylist()
|
|
except Exception:
|
|
return summary
|
|
|
|
exposure_rows = [row for row in rows if float(row["equity"]) > 0]
|
|
if not exposure_rows:
|
|
return summary
|
|
|
|
if summary.get("avg_gross_exposure_pct") is None:
|
|
summary["avg_gross_exposure_pct"] = sum(
|
|
float(row["gross_exposure"]) / float(row["equity"]) * 100.0
|
|
for row in exposure_rows
|
|
) / len(exposure_rows)
|
|
if summary.get("avg_net_exposure_pct") is None:
|
|
summary["avg_net_exposure_pct"] = sum(
|
|
float(row["net_exposure"]) / float(row["equity"]) * 100.0
|
|
for row in exposure_rows
|
|
) / len(exposure_rows)
|
|
if summary.get("days_in_market_pct") is None:
|
|
summary["days_in_market_pct"] = (
|
|
sum(1 for row in rows if float(row["gross_exposure"]) > 0) / len(rows) * 100.0
|
|
)
|
|
|
|
return summary
|
|
|
|
|
|
def _hydrate_split_result(result: SplitResult | None) -> SplitResult | None:
|
|
if result is None:
|
|
return None
|
|
needed_fields = (
|
|
"annualized_return_pct",
|
|
"avg_gross_exposure_pct",
|
|
"avg_net_exposure_pct",
|
|
"days_in_market_pct",
|
|
)
|
|
if all(getattr(result, field) is not None for field in needed_fields):
|
|
return result
|
|
summary = _load_run_metrics_summary(result.run_id)
|
|
if summary is None:
|
|
return result
|
|
updates = {
|
|
field: summary.get(field)
|
|
for field in needed_fields
|
|
if getattr(result, field) is None and summary.get(field) is not None
|
|
}
|
|
if not updates:
|
|
return result
|
|
return result.model_copy(update=updates)
|
|
|
|
|
|
def _compute_split_quality_score(
|
|
result: SplitResult | None,
|
|
) -> tuple[float | None, dict[str, float], str | None]:
|
|
metrics = _metrics_from_split_result(_hydrate_split_result(result))
|
|
if metrics is None:
|
|
return None, {}, None
|
|
sqs_v2_score, sqs_v2_breakdown = compute_sqs_v2(metrics)
|
|
if sqs_v2_score is not None:
|
|
return sqs_v2_score, sqs_v2_breakdown, "sqs_v2"
|
|
sqs_score, sqs_breakdown = compute_sqs(metrics)
|
|
return sqs_score, sqs_breakdown, "sqs"
|
|
|
|
|
|
def compute_unified_split_quality(
|
|
metrics: MetricsBundle,
|
|
weights: UnifiedScoreWeights | None = None,
|
|
) -> tuple[float | None, dict[str, float]]:
|
|
"""Compute a stricter split-level quality score."""
|
|
if metrics.avg_gross_exposure_pct is None or metrics.days_in_market_pct is None:
|
|
return None, {}
|
|
|
|
w = weights or _DEFAULT_UNIFIED_WEIGHTS
|
|
|
|
pf_score = _normalize(metrics.profit_factor, low=1.0, high=3.0)
|
|
ret_score = _normalize(metrics.total_return_pct, low=0.0, high=3.0)
|
|
profitability = pf_score * 0.5 + ret_score * 0.5
|
|
|
|
dd_score = _normalize_inverse(metrics.max_drawdown_pct, low=5.0, high=0.5)
|
|
sharpe_score = _normalize(metrics.sharpe_ratio, low=0.0, high=3.5)
|
|
risk = dd_score * 0.5 + sharpe_score * 0.5
|
|
|
|
wr_score = _normalize(metrics.win_rate, low=0.50, high=0.65)
|
|
mwr_score = _normalize(metrics.monthly_win_rate, low=0.45, high=0.75)
|
|
consistency = wr_score * 0.5 + mwr_score * 0.5
|
|
|
|
r2_score = _normalize(metrics.equity_curve_r_squared, low=0.10, high=0.90)
|
|
tc_score = _normalize(float(metrics.trade_count), low=20.0, high=80.0)
|
|
robustness = r2_score * 0.5 + tc_score * 0.5
|
|
|
|
return_on_gross_exposure = None
|
|
if metrics.total_return_pct is not None and metrics.avg_gross_exposure_pct > 0:
|
|
return_on_gross_exposure = metrics.total_return_pct / metrics.avg_gross_exposure_pct
|
|
roe_score = _normalize(return_on_gross_exposure, low=0.05, high=0.40)
|
|
dim_score = _normalize_band(
|
|
metrics.days_in_market_pct,
|
|
low_bad=20.0,
|
|
low_good=45.0,
|
|
high_good=75.0,
|
|
high_bad=90.0,
|
|
)
|
|
capital_efficiency = roe_score * 0.7 + dim_score * 0.3
|
|
|
|
split_quality = (
|
|
profitability * w.split_profitability
|
|
+ risk * w.split_risk
|
|
+ consistency * w.split_consistency
|
|
+ robustness * w.split_robustness
|
|
+ capital_efficiency * w.split_capital_efficiency
|
|
)
|
|
if metrics.trade_count < 20:
|
|
split_quality *= 0.75
|
|
|
|
split_quality = round(split_quality, 1)
|
|
breakdown = {
|
|
"profitability": round(profitability, 1),
|
|
"risk": round(risk, 1),
|
|
"consistency": round(consistency, 1),
|
|
"robustness": round(robustness, 1),
|
|
"capital_efficiency": round(capital_efficiency, 1),
|
|
}
|
|
return split_quality, breakdown
|
|
|
|
|
|
def compute_promotion_score(
|
|
test_result: SplitResult | None,
|
|
valid_result: SplitResult | None,
|
|
weights: PromotionScoreWeights | None = None,
|
|
) -> tuple[float | None, dict[str, float]]:
|
|
"""Compute a promotion score using valid/test quality and a floor term."""
|
|
test_score, _, _ = _compute_split_quality_score(test_result)
|
|
valid_score, _, _ = _compute_split_quality_score(valid_result)
|
|
if test_score is None or valid_score is None:
|
|
return None, {}
|
|
|
|
w = weights or _DEFAULT_PROMOTION_WEIGHTS
|
|
floor_score = min(test_score, valid_score)
|
|
promotion_score = (
|
|
valid_score * w.valid_quality
|
|
+ test_score * w.test_quality
|
|
+ floor_score * w.floor_quality
|
|
)
|
|
breakdown = {
|
|
"valid_quality": round(valid_score, 1),
|
|
"test_quality": round(test_score, 1),
|
|
"floor_quality": round(floor_score, 1),
|
|
}
|
|
return round(promotion_score, 1), breakdown
|
|
|
|
|
|
def compute_unified_score(
|
|
test_result: SplitResult | None,
|
|
valid_result: SplitResult | None,
|
|
weights: UnifiedScoreWeights | None = None,
|
|
) -> tuple[float | None, dict[str, float]]:
|
|
"""Compute one integrated score for real promotion decisions."""
|
|
test_result = _hydrate_split_result(test_result)
|
|
valid_result = _hydrate_split_result(valid_result)
|
|
test_metrics = _metrics_from_split_result(test_result)
|
|
valid_metrics = _metrics_from_split_result(valid_result)
|
|
if test_metrics is None or valid_metrics is None:
|
|
return None, {}
|
|
|
|
w = weights or _DEFAULT_UNIFIED_WEIGHTS
|
|
test_quality, _ = compute_unified_split_quality(test_metrics, w)
|
|
valid_quality, _ = compute_unified_split_quality(valid_metrics, w)
|
|
if test_quality is None or valid_quality is None:
|
|
return None, {}
|
|
|
|
floor_quality = min(test_quality, valid_quality)
|
|
gap_quality = _normalize_inverse(abs(valid_quality - test_quality), low=35.0, high=5.0)
|
|
unified_score = (
|
|
valid_quality * w.valid_quality
|
|
+ test_quality * w.test_quality
|
|
+ floor_quality * w.floor_quality
|
|
+ gap_quality * w.gap_quality
|
|
)
|
|
breakdown = {
|
|
"valid_quality": round(valid_quality, 1),
|
|
"test_quality": round(test_quality, 1),
|
|
"floor_quality": round(floor_quality, 1),
|
|
"gap_quality": round(gap_quality, 1),
|
|
}
|
|
return _calibrate_sqs(unified_score), breakdown
|
|
|
|
|
|
def compute_public_sqs(
|
|
train_result: SplitResult | None,
|
|
valid_result: SplitResult | None,
|
|
test_result: SplitResult | None,
|
|
walk_forward_summary: WalkForwardSummary | None = None,
|
|
robustness_matrix_summary: RobustnessMatrixSummary | None = None,
|
|
out_of_time_robustness_summary: RobustnessMatrixSummary | None = None,
|
|
common_window_summary: CommonWindowSummary | None = None,
|
|
rqs_score: float | None = None,
|
|
wfqs_score: float | None = None,
|
|
deployment_score: float | None = None,
|
|
) -> tuple[float | None, dict[str, float], str | None]:
|
|
"""Return the current public-facing SQS.
|
|
|
|
v4 keeps the v3 stress/robustness gates intact, and adds a modest
|
|
full-cycle capital-growth term when a comparable common-window run
|
|
is available. Missing common-window summaries fall back to v3.
|
|
"""
|
|
return compute_public_sqs_v4(
|
|
train_result,
|
|
valid_result,
|
|
test_result,
|
|
walk_forward_summary=walk_forward_summary,
|
|
robustness_matrix_summary=robustness_matrix_summary,
|
|
out_of_time_robustness_summary=out_of_time_robustness_summary,
|
|
common_window_summary=common_window_summary,
|
|
rqs_score=rqs_score,
|
|
wfqs_v2_score=wfqs_score,
|
|
)
|
|
|
|
|
|
def _compute_return_split_score(
|
|
metrics: MetricsBundle,
|
|
split_name: str,
|
|
weights: ReturnScoreWeights | None = None,
|
|
) -> tuple[float, dict[str, float]]:
|
|
"""Compute a return-max score for a single split."""
|
|
w = weights or _DEFAULT_RQS_WEIGHTS
|
|
|
|
return_bands = {
|
|
"train": (0.0, 50.0),
|
|
"valid": (0.0, 30.0),
|
|
"test": (0.0, 35.0),
|
|
}
|
|
annualized_return_bands = {
|
|
"train": (0.0, 25.0),
|
|
"valid": (0.0, 120.0),
|
|
"test": (0.0, 120.0),
|
|
}
|
|
dd_bands = {
|
|
"train": (20.0, 2.0),
|
|
"valid": (12.0, 2.0),
|
|
"test": (12.0, 2.0),
|
|
}
|
|
ret_low, ret_high = return_bands.get(split_name, (0.0, 35.0))
|
|
ann_low, ann_high = annualized_return_bands.get(split_name, (0.0, 100.0))
|
|
dd_low, dd_high = dd_bands.get(split_name, (12.0, 2.0))
|
|
|
|
total_return_score = _normalize(metrics.total_return_pct, low=ret_low, high=ret_high)
|
|
annualized_return_score = _normalize(metrics.annualized_return_pct, low=ann_low, high=ann_high)
|
|
effective_profit_factor = metrics.profit_factor
|
|
if (
|
|
effective_profit_factor is None
|
|
and metrics.trade_count > 0
|
|
and metrics.win_rate is not None
|
|
and metrics.win_rate >= 0.999
|
|
):
|
|
# Backtest summaries emit ``None`` when there are no losing trades.
|
|
# For return-max ranking that should be treated as capped-best, not zero.
|
|
effective_profit_factor = 3.0
|
|
pf_score = _normalize(effective_profit_factor, low=1.0, high=3.0)
|
|
sharpe_score = _normalize(metrics.sharpe_ratio, low=0.0, high=3.5)
|
|
drawdown_score = _normalize_inverse(metrics.max_drawdown_pct, low=dd_low, high=dd_high)
|
|
gross_score = _normalize_band(
|
|
metrics.avg_gross_exposure_pct,
|
|
low_bad=2.0,
|
|
low_good=8.0,
|
|
high_good=40.0,
|
|
high_bad=75.0,
|
|
)
|
|
dim_score = _normalize_band(
|
|
metrics.days_in_market_pct,
|
|
low_bad=5.0,
|
|
low_good=15.0,
|
|
high_good=60.0,
|
|
high_bad=90.0,
|
|
)
|
|
return_on_gross_exposure = None
|
|
if metrics.total_return_pct is not None and metrics.avg_gross_exposure_pct and metrics.avg_gross_exposure_pct > 0:
|
|
return_on_gross_exposure = metrics.total_return_pct / metrics.avg_gross_exposure_pct
|
|
return_on_gross_score = _normalize(return_on_gross_exposure, low=0.20, high=2.50)
|
|
|
|
split_score = (
|
|
total_return_score * w.split_total_return
|
|
+ annualized_return_score * w.split_annualized_return
|
|
+ pf_score * w.split_profitability
|
|
+ sharpe_score * w.split_sharpe
|
|
+ drawdown_score * w.split_drawdown
|
|
+ return_on_gross_score * w.split_return_on_gross
|
|
+ gross_score * w.split_gross_exposure
|
|
+ dim_score * w.split_days_in_market
|
|
)
|
|
if metrics.trade_count < w.low_trade_penalty_threshold:
|
|
split_score *= w.low_trade_penalty_factor
|
|
|
|
split_score = round(split_score, 1)
|
|
breakdown = {
|
|
"total_return": round(total_return_score, 1),
|
|
"annualized_return": round(annualized_return_score, 1),
|
|
"profitability": round(pf_score, 1),
|
|
"sharpe": round(sharpe_score, 1),
|
|
"drawdown": round(drawdown_score, 1),
|
|
"return_on_gross": round(return_on_gross_score, 1),
|
|
"gross_exposure": round(gross_score, 1),
|
|
"days_in_market": round(dim_score, 1),
|
|
}
|
|
return split_score, breakdown
|
|
|
|
|
|
def compute_rqs(
|
|
train_result: SplitResult | None,
|
|
valid_result: SplitResult | None,
|
|
test_result: SplitResult | None,
|
|
weights: ReturnScoreWeights | None = None,
|
|
) -> tuple[float | None, dict[str, float]]:
|
|
"""Compute Return Quality Score for return-max strategy ranking."""
|
|
w = weights or _DEFAULT_RQS_WEIGHTS
|
|
|
|
hydrated = {
|
|
"train": _hydrate_split_result(train_result),
|
|
"valid": _hydrate_split_result(valid_result),
|
|
"test": _hydrate_split_result(test_result),
|
|
}
|
|
split_scores: dict[str, float] = {}
|
|
for split_name, result in hydrated.items():
|
|
metrics = _metrics_from_split_result(result)
|
|
if metrics is None:
|
|
continue
|
|
split_score, _ = _compute_return_split_score(metrics, split_name, w)
|
|
split_scores[split_name] = split_score
|
|
|
|
if "valid" not in split_scores or "test" not in split_scores:
|
|
return None, {}
|
|
|
|
weighted_sum = 0.0
|
|
weighted_den = 0.0
|
|
if "train" in split_scores:
|
|
weighted_sum += split_scores["train"] * w.train_quality
|
|
weighted_den += w.train_quality
|
|
weighted_sum += split_scores["valid"] * w.valid_quality
|
|
weighted_den += w.valid_quality
|
|
weighted_sum += split_scores["test"] * w.test_quality
|
|
weighted_den += w.test_quality
|
|
split_quality = weighted_sum / weighted_den if weighted_den > 0 else 0.0
|
|
|
|
floor_quality = min(split_scores.values())
|
|
gap_quality = _normalize_inverse(
|
|
max(split_scores.values()) - min(split_scores.values()),
|
|
low=60.0,
|
|
high=10.0,
|
|
)
|
|
rqs = split_quality * (1.0 - w.floor_quality - w.gap_quality) + floor_quality * w.floor_quality + gap_quality * w.gap_quality
|
|
if "train" not in split_scores:
|
|
rqs *= w.missing_train_penalty
|
|
|
|
breakdown = {
|
|
"train_quality": round(split_scores.get("train", 0.0), 1),
|
|
"valid_quality": round(split_scores["valid"], 1),
|
|
"test_quality": round(split_scores["test"], 1),
|
|
"floor_quality": round(floor_quality, 1),
|
|
"gap_quality": round(gap_quality, 1),
|
|
}
|
|
return round(rqs, 1), breakdown
|
|
|
|
|
|
def compute_wfqs(
|
|
walk_forward_summary: WalkForwardSummary | None,
|
|
weights: WalkForwardScoreWeights | None = None,
|
|
) -> tuple[float | None, dict[str, float]]:
|
|
"""Compute a walk-forward robustness score from fold-level aggregates."""
|
|
if walk_forward_summary is None or walk_forward_summary.fold_count <= 0:
|
|
return None, {}
|
|
|
|
w = weights or _DEFAULT_WFQS_WEIGHTS
|
|
test = walk_forward_summary.test_aggregate
|
|
gap = walk_forward_summary.gap_stats
|
|
|
|
median_return_score = _normalize(test.median_return_pct, low=0.0, high=25.0)
|
|
mean_return_score = _normalize(test.mean_return_pct, low=0.0, high=25.0)
|
|
worst_return_score = _normalize(test.worst_return_pct, low=-5.0, high=10.0)
|
|
positive_fold_rate_score = _normalize(test.positive_fold_rate_pct, low=50.0, high=100.0)
|
|
profit_factor_score = _normalize(test.mean_profit_factor, low=1.0, high=4.0)
|
|
drawdown_score = _normalize_inverse(test.mean_max_drawdown_pct, low=15.0, high=2.0)
|
|
gap_score = _normalize_inverse(gap.mean_train_test_return_gap_pct, low=70.0, high=15.0)
|
|
fold_count_score = _normalize(float(walk_forward_summary.fold_count), low=4.0, high=10.0)
|
|
|
|
wfqs = (
|
|
median_return_score * w.median_return
|
|
+ mean_return_score * w.mean_return
|
|
+ worst_return_score * w.worst_return
|
|
+ positive_fold_rate_score * w.positive_fold_rate
|
|
+ profit_factor_score * w.profit_factor
|
|
+ drawdown_score * w.drawdown
|
|
+ gap_score * w.train_test_gap
|
|
+ fold_count_score * w.fold_count
|
|
)
|
|
if walk_forward_summary.fold_count < w.low_fold_penalty_threshold:
|
|
wfqs *= w.low_fold_penalty_factor
|
|
|
|
breakdown = {
|
|
"median_return": round(median_return_score, 1),
|
|
"mean_return": round(mean_return_score, 1),
|
|
"worst_return": round(worst_return_score, 1),
|
|
"positive_fold_rate": round(positive_fold_rate_score, 1),
|
|
"profit_factor": round(profit_factor_score, 1),
|
|
"drawdown": round(drawdown_score, 1),
|
|
"train_test_gap": round(gap_score, 1),
|
|
"fold_count": round(fold_count_score, 1),
|
|
}
|
|
return round(wfqs, 1), breakdown
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# WFQS v2: multiplicative penalty model
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _gap_penalty(gap_pct: float | None) -> float:
|
|
"""Multiplicative penalty based on mean train-test gap percentage."""
|
|
if gap_pct is None:
|
|
return 1.0
|
|
if gap_pct <= 30.0:
|
|
return 1.0
|
|
if gap_pct <= 100.0:
|
|
return 1.0 - (gap_pct - 30.0) / (100.0 - 30.0) * 0.4 # 1.0 → 0.6
|
|
if gap_pct <= 300.0:
|
|
return 0.6 - (gap_pct - 100.0) / (300.0 - 100.0) * 0.3 # 0.6 → 0.3
|
|
return 0.2
|
|
|
|
|
|
def _fold_variance_penalty(fold_return_cv: float | None) -> float:
|
|
"""Multiplicative penalty based on CV of fold test returns."""
|
|
if fold_return_cv is None:
|
|
return 1.0
|
|
if fold_return_cv <= 0.5:
|
|
return 1.0
|
|
if fold_return_cv <= 1.5:
|
|
return 1.0 - (fold_return_cv - 0.5) / (1.5 - 0.5) * 0.3 # 1.0 → 0.7
|
|
return 0.6
|
|
|
|
|
|
def _trade_credibility(mean_trades: float | None, mean_win_rate: float | None) -> float:
|
|
"""Multiplicative penalty for low trade counts or suspiciously high win rates."""
|
|
if mean_trades is None:
|
|
return 0.5
|
|
if mean_trades < 5:
|
|
factor = 0.5
|
|
elif mean_trades < 10:
|
|
factor = 0.7
|
|
elif mean_trades < 20:
|
|
factor = 0.85
|
|
else:
|
|
factor = 1.0
|
|
if (mean_win_rate or 0.0) > 0.95 and mean_trades < 15:
|
|
factor *= 0.8
|
|
return factor
|
|
|
|
|
|
def _engine_reliability_penalty(ratio: float | None) -> float:
|
|
"""Multiplicative penalty based on engine reliability ratio."""
|
|
if ratio is None:
|
|
return 1.0
|
|
if ratio >= 0.7:
|
|
return 1.0
|
|
if ratio >= 0.3:
|
|
return 0.5 + (ratio - 0.3) / (0.7 - 0.3) * 0.5 # 0.5 → 1.0
|
|
return 0.4
|
|
|
|
|
|
def _derive_fold_stats(
|
|
walk_forward_summary: WalkForwardSummary,
|
|
) -> tuple[float | None, float | None, float | None]:
|
|
"""Derive mean_trade_count, mean_win_rate, fold_return_cv from folds when aggregate fields are missing."""
|
|
import statistics as _stats
|
|
|
|
folds = walk_forward_summary.folds
|
|
if not folds:
|
|
return None, None, None
|
|
|
|
trade_counts = [float(f.test_metrics.trade_count) for f in folds]
|
|
win_rates = [f.test_metrics.win_rate for f in folds if f.test_metrics.win_rate is not None]
|
|
returns = [f.test_metrics.total_return_pct for f in folds if f.test_metrics.total_return_pct is not None]
|
|
|
|
mean_tc = round(_stats.mean(trade_counts), 1) if trade_counts else None
|
|
mean_wr = round(_stats.mean(win_rates), 4) if win_rates else None
|
|
cv = None
|
|
if len(returns) >= 2:
|
|
mean_ret = _stats.mean(returns)
|
|
if abs(mean_ret) > 1e-9:
|
|
cv = round(_stats.stdev(returns) / abs(mean_ret), 3)
|
|
return mean_tc, mean_wr, cv
|
|
|
|
|
|
def _wfqs_v2_quality_from_aggregate(
|
|
aggregate: WalkForwardAggregate,
|
|
fold_count: int,
|
|
weights: WFQSv2Weights,
|
|
) -> float:
|
|
"""Compute additive WFQS v2 quality score before multiplicative penalties."""
|
|
median_return_score = _normalize(aggregate.median_return_pct, low=0.0, high=25.0)
|
|
mean_return_score = _normalize(aggregate.mean_return_pct, low=0.0, high=25.0)
|
|
worst_return_score = _normalize(aggregate.worst_return_pct, low=-5.0, high=10.0)
|
|
positive_fold_rate_score = _normalize(aggregate.positive_fold_rate_pct, low=50.0, high=100.0)
|
|
profit_factor_score = _normalize(aggregate.mean_profit_factor, low=1.0, high=4.0)
|
|
drawdown_score = _normalize_inverse(aggregate.mean_max_drawdown_pct, low=15.0, high=2.0)
|
|
fold_count_score = _normalize(float(fold_count), low=4.0, high=10.0)
|
|
quality = (
|
|
median_return_score * weights.median_return
|
|
+ mean_return_score * weights.mean_return
|
|
+ worst_return_score * weights.worst_return
|
|
+ positive_fold_rate_score * weights.positive_fold_rate
|
|
+ profit_factor_score * weights.profit_factor
|
|
+ drawdown_score * weights.drawdown
|
|
+ fold_count_score * weights.fold_count
|
|
)
|
|
if fold_count < weights.low_fold_penalty_threshold:
|
|
quality *= weights.low_fold_penalty_factor
|
|
return quality
|
|
|
|
|
|
def _build_recent_fold_aggregate(
|
|
walk_forward_summary: WalkForwardSummary,
|
|
lookback_days: int,
|
|
min_folds: int,
|
|
) -> tuple[WalkForwardAggregate | None, int]:
|
|
"""Aggregate test-fold quality for folds ending within the recent lookback window."""
|
|
import statistics as _stats
|
|
|
|
folds = walk_forward_summary.folds
|
|
if not folds:
|
|
return None, 0
|
|
|
|
anchor = max(f.test_end for f in folds)
|
|
cutoff = anchor - dt.timedelta(days=max(lookback_days - 1, 0))
|
|
recent_folds = [f for f in folds if f.test_end >= cutoff]
|
|
if len(recent_folds) < min_folds:
|
|
return None, len(recent_folds)
|
|
|
|
returns = [f.test_metrics.total_return_pct for f in recent_folds if f.test_metrics.total_return_pct is not None]
|
|
pfs = [f.test_metrics.profit_factor for f in recent_folds if f.test_metrics.profit_factor is not None]
|
|
dds = [f.test_metrics.max_drawdown_pct for f in recent_folds if f.test_metrics.max_drawdown_pct is not None]
|
|
trade_counts = [float(f.test_metrics.trade_count) for f in recent_folds]
|
|
win_rates = [f.test_metrics.win_rate for f in recent_folds if f.test_metrics.win_rate is not None]
|
|
positive = [r for r in returns if r > 0]
|
|
|
|
aggregate = WalkForwardAggregate(
|
|
mean_return_pct=round(_stats.mean(returns), 2) if returns else None,
|
|
median_return_pct=round(_stats.median(returns), 2) if returns else None,
|
|
worst_return_pct=round(min(returns), 2) if returns else None,
|
|
positive_fold_rate_pct=round(len(positive) / len(recent_folds) * 100.0, 1) if recent_folds else None,
|
|
mean_profit_factor=round(_stats.mean(pfs), 2) if pfs else None,
|
|
mean_max_drawdown_pct=round(_stats.mean(dds), 2) if dds else None,
|
|
mean_trade_count=round(_stats.mean(trade_counts), 1) if trade_counts else None,
|
|
mean_win_rate=round(_stats.mean(win_rates), 4) if win_rates else None,
|
|
)
|
|
return aggregate, len(recent_folds)
|
|
|
|
|
|
def _public_activity_factor(
|
|
test_result: SplitResult | None,
|
|
walk_forward_summary: WalkForwardSummary | None,
|
|
) -> tuple[float, dict[str, float]]:
|
|
"""Return a light multiplicative penalty for strategies that barely trade."""
|
|
if test_result is None or walk_forward_summary is None:
|
|
return 1.0, {}
|
|
|
|
mean_tc = walk_forward_summary.test_aggregate.mean_trade_count
|
|
if mean_tc is None:
|
|
derived_tc, _, _ = _derive_fold_stats(walk_forward_summary)
|
|
mean_tc = derived_tc
|
|
|
|
test_trade_score = _normalize(float(test_result.trade_count), low=8.0, high=24.0)
|
|
wf_trade_score = _normalize(mean_tc, low=3.0, high=10.0)
|
|
dim_score = _normalize(test_result.days_in_market_pct, low=20.0, high=60.0)
|
|
|
|
activity_quality = (
|
|
test_trade_score * 0.45
|
|
+ wf_trade_score * 0.35
|
|
+ dim_score * 0.20
|
|
)
|
|
|
|
if test_result.trade_count < 10:
|
|
activity_quality *= 0.85
|
|
if (mean_tc or 0.0) < 4.0:
|
|
activity_quality *= 0.85
|
|
|
|
factor = 0.50 + 0.50 * (activity_quality / 100.0)
|
|
breakdown = {
|
|
"activity_factor": round(factor, 2),
|
|
"activity_quality": round(activity_quality, 1),
|
|
"activity_test_trades": round(test_trade_score, 1),
|
|
"activity_wf_mean_trades": round(wf_trade_score, 1),
|
|
"activity_days_in_market": round(dim_score, 1),
|
|
}
|
|
return factor, breakdown
|
|
|
|
|
|
def compute_wfqs_v2(
|
|
walk_forward_summary: WalkForwardSummary | None,
|
|
weights: WFQSv2Weights | None = None,
|
|
) -> tuple[float | None, dict[str, float]]:
|
|
"""Compute WFQS v2 with multiplicative overfitting penalties."""
|
|
if walk_forward_summary is None or walk_forward_summary.fold_count <= 0:
|
|
return None, {}
|
|
|
|
w = weights or _DEFAULT_WFQS_V2_WEIGHTS
|
|
test = walk_forward_summary.test_aggregate
|
|
gap = walk_forward_summary.gap_stats
|
|
|
|
# Derive missing stats from folds when aggregate fields are absent
|
|
mean_tc = test.mean_trade_count
|
|
mean_wr = test.mean_win_rate
|
|
fold_cv = gap.fold_return_cv
|
|
if mean_tc is None or mean_wr is None or fold_cv is None:
|
|
derived_tc, derived_wr, derived_cv = _derive_fold_stats(walk_forward_summary)
|
|
if mean_tc is None:
|
|
mean_tc = derived_tc
|
|
if mean_wr is None:
|
|
mean_wr = derived_wr
|
|
if fold_cv is None:
|
|
fold_cv = derived_cv
|
|
|
|
overall_quality = _wfqs_v2_quality_from_aggregate(
|
|
test,
|
|
walk_forward_summary.fold_count,
|
|
w,
|
|
)
|
|
median_return_score = _normalize(test.median_return_pct, low=0.0, high=25.0)
|
|
mean_return_score = _normalize(test.mean_return_pct, low=0.0, high=25.0)
|
|
worst_return_score = _normalize(test.worst_return_pct, low=-5.0, high=10.0)
|
|
positive_fold_rate_score = _normalize(test.positive_fold_rate_pct, low=50.0, high=100.0)
|
|
profit_factor_score = _normalize(test.mean_profit_factor, low=1.0, high=4.0)
|
|
drawdown_score = _normalize_inverse(test.mean_max_drawdown_pct, low=15.0, high=2.0)
|
|
fold_count_score = _normalize(float(walk_forward_summary.fold_count), low=4.0, high=10.0)
|
|
recent_aggregate, recent_fold_count = _build_recent_fold_aggregate(
|
|
walk_forward_summary,
|
|
lookback_days=w.recent_lookback_days,
|
|
min_folds=w.recent_min_folds,
|
|
)
|
|
recent_quality = None
|
|
if recent_aggregate is not None:
|
|
recent_quality = _wfqs_v2_quality_from_aggregate(
|
|
recent_aggregate,
|
|
recent_fold_count,
|
|
w,
|
|
)
|
|
if recent_quality is not None:
|
|
base_score = overall_quality * (1.0 - w.recent_fold_quality) + recent_quality * w.recent_fold_quality
|
|
else:
|
|
base_score = overall_quality
|
|
|
|
# Multiplicative penalties
|
|
gp = _gap_penalty(gap.mean_train_test_return_gap_pct)
|
|
fvp = _fold_variance_penalty(fold_cv)
|
|
tc = _trade_credibility(mean_tc, mean_wr)
|
|
er = _engine_reliability_penalty(walk_forward_summary.engine_reliability_ratio)
|
|
|
|
wfqs_v2 = base_score * gp * fvp * tc * er
|
|
|
|
breakdown = {
|
|
"base_score": round(base_score, 1),
|
|
"median_return": round(median_return_score, 1),
|
|
"mean_return": round(mean_return_score, 1),
|
|
"worst_return": round(worst_return_score, 1),
|
|
"positive_fold_rate": round(positive_fold_rate_score, 1),
|
|
"profit_factor": round(profit_factor_score, 1),
|
|
"drawdown": round(drawdown_score, 1),
|
|
"fold_count": round(fold_count_score, 1),
|
|
"overall_quality": round(overall_quality, 1),
|
|
"recent_quality": round(recent_quality, 1) if recent_quality is not None else 0.0,
|
|
"recent_fold_count": float(recent_fold_count),
|
|
"recent_fold_weight": round(w.recent_fold_quality if recent_quality is not None else 0.0, 2),
|
|
"gap_penalty": round(gp, 3),
|
|
"fold_variance_penalty": round(fvp, 3),
|
|
"trade_credibility": round(tc, 3),
|
|
"engine_reliability": round(er, 3),
|
|
}
|
|
return round(wfqs_v2, 1), breakdown
|
|
|
|
|
|
def _compute_public_sqs_components(
|
|
train_result: SplitResult | None,
|
|
valid_result: SplitResult | None,
|
|
test_result: SplitResult | None,
|
|
walk_forward_summary: WalkForwardSummary | None = None,
|
|
robustness_matrix_summary: RobustnessMatrixSummary | None = None,
|
|
out_of_time_robustness_summary: RobustnessMatrixSummary | None = None,
|
|
rqs_score: float | None = None,
|
|
wfqs_v2_score: float | None = None,
|
|
) -> tuple[dict[str, Any] | None, dict[str, float], str | None]:
|
|
"""Resolve shared components for public SQS variants."""
|
|
resolved_rqs = rqs_score
|
|
if resolved_rqs is None:
|
|
resolved_rqs, _ = compute_rqs(train_result, valid_result, test_result)
|
|
if resolved_rqs is None:
|
|
return None, {}, None
|
|
|
|
missing_requirements: dict[str, float] = {}
|
|
if walk_forward_summary is None:
|
|
missing_requirements["requires_walk_forward"] = 1.0
|
|
if robustness_matrix_summary is None:
|
|
missing_requirements["requires_robustness"] = 1.0
|
|
if out_of_time_robustness_summary is None:
|
|
missing_requirements["requires_out_of_time_robustness"] = 1.0
|
|
if missing_requirements:
|
|
return None, missing_requirements, "pending_validation"
|
|
|
|
resolved_wfqs_v2 = wfqs_v2_score
|
|
if resolved_wfqs_v2 is None:
|
|
resolved_wfqs_v2, _ = compute_wfqs_v2(walk_forward_summary)
|
|
|
|
if resolved_wfqs_v2 is None:
|
|
return None, {}, None
|
|
|
|
test = walk_forward_summary.test_aggregate
|
|
gap = walk_forward_summary.gap_stats
|
|
pass_positive = (test.positive_fold_rate_pct or 0.0) >= 70.0
|
|
pass_median = (test.median_return_pct or 0.0) >= 5.0
|
|
pass_worst = (test.worst_return_pct or 0.0) >= -5.0
|
|
pass_gap = (gap.mean_train_test_return_gap_pct or 999.0) <= 35.0
|
|
pass_count = sum([pass_positive, pass_median, pass_worst, pass_gap])
|
|
deployment_gate_factor = {4: 1.00, 3: 0.85, 2: 0.65, 1: 0.40, 0: 0.20}[pass_count]
|
|
base_score = (resolved_rqs * 0.45 + resolved_wfqs_v2 * 0.55) * deployment_gate_factor
|
|
|
|
gate_factor_rb, gate_breakdown = compute_robustness_gate(robustness_matrix_summary)
|
|
oot_gate_factor, oot_gate_breakdown = compute_oot_robustness_gate(out_of_time_robustness_summary)
|
|
oot_quality, oot_quality_breakdown = compute_oot_robustness_quality(out_of_time_robustness_summary)
|
|
activity_factor, activity_breakdown = _public_activity_factor(test_result, walk_forward_summary)
|
|
return (
|
|
{
|
|
"base_score": base_score,
|
|
"deployment_gate_factor": deployment_gate_factor,
|
|
"rb_gate_factor": gate_factor_rb,
|
|
"rb_gate_breakdown": gate_breakdown,
|
|
"oot_gate_factor": oot_gate_factor,
|
|
"oot_gate_breakdown": oot_gate_breakdown,
|
|
"oot_quality": oot_quality,
|
|
"oot_quality_breakdown": oot_quality_breakdown,
|
|
"activity_factor": activity_factor,
|
|
"activity_breakdown": activity_breakdown,
|
|
},
|
|
{},
|
|
"v3_deployment",
|
|
)
|
|
|
|
|
|
def compute_public_sqs_v2(
|
|
train_result: SplitResult | None,
|
|
valid_result: SplitResult | None,
|
|
test_result: SplitResult | None,
|
|
walk_forward_summary: WalkForwardSummary | None = None,
|
|
robustness_matrix_summary: RobustnessMatrixSummary | None = None,
|
|
out_of_time_robustness_summary: RobustnessMatrixSummary | None = None,
|
|
rqs_score: float | None = None,
|
|
wfqs_v2_score: float | None = None,
|
|
) -> tuple[float | None, dict[str, float], str | None]:
|
|
"""Return legacy stress-adjusted public SQS.
|
|
|
|
This keeps the historical multiplicative OOT quality factor that
|
|
compresses strong recent strategies when stress-period quality is weaker.
|
|
"""
|
|
components, breakdown, source = _compute_public_sqs_components(
|
|
train_result,
|
|
valid_result,
|
|
test_result,
|
|
walk_forward_summary=walk_forward_summary,
|
|
robustness_matrix_summary=robustness_matrix_summary,
|
|
out_of_time_robustness_summary=out_of_time_robustness_summary,
|
|
rqs_score=rqs_score,
|
|
wfqs_v2_score=wfqs_v2_score,
|
|
)
|
|
if components is None:
|
|
return None, breakdown, source
|
|
|
|
oot_quality_factor = 1.0
|
|
if components["oot_quality"] is not None:
|
|
oot_quality_factor = 0.30 + 0.70 * (components["oot_quality"] / 100.0)
|
|
final_score = (
|
|
components["base_score"]
|
|
* components["rb_gate_factor"]
|
|
* components["oot_gate_factor"]
|
|
* oot_quality_factor
|
|
* components["activity_factor"]
|
|
)
|
|
resolved_breakdown = {
|
|
"base_score": round(components["base_score"], 1),
|
|
"gate_factor": round(components["rb_gate_factor"], 2),
|
|
**components["rb_gate_breakdown"],
|
|
"oot_gate_factor": round(components["oot_gate_factor"], 2),
|
|
"oot_quality_factor": round(oot_quality_factor, 2),
|
|
**{
|
|
f"oot_{key}": value
|
|
for key, value in components["oot_gate_breakdown"].items()
|
|
},
|
|
**{
|
|
f"oot_{key}": value
|
|
for key, value in components["oot_quality_breakdown"].items()
|
|
},
|
|
**components["activity_breakdown"],
|
|
}
|
|
if components["oot_quality"] is not None:
|
|
resolved_breakdown["oot_quality"] = round(components["oot_quality"], 1)
|
|
return round(final_score, 1), resolved_breakdown, "v2_deployment+robustness+oot"
|
|
|
|
|
|
def compute_public_sqs_v3(
|
|
train_result: SplitResult | None,
|
|
valid_result: SplitResult | None,
|
|
test_result: SplitResult | None,
|
|
walk_forward_summary: WalkForwardSummary | None = None,
|
|
robustness_matrix_summary: RobustnessMatrixSummary | None = None,
|
|
out_of_time_robustness_summary: RobustnessMatrixSummary | None = None,
|
|
rqs_score: float | None = None,
|
|
wfqs_v2_score: float | None = None,
|
|
) -> tuple[float | None, dict[str, float], str | None]:
|
|
"""Return public SQS v3.
|
|
|
|
v3 treats stress OOT as a hard gate for eligibility, but keeps OOT quality
|
|
as a diagnostic instead of a multiplicative ranking penalty.
|
|
"""
|
|
components, breakdown, source = _compute_public_sqs_components(
|
|
train_result,
|
|
valid_result,
|
|
test_result,
|
|
walk_forward_summary=walk_forward_summary,
|
|
robustness_matrix_summary=robustness_matrix_summary,
|
|
out_of_time_robustness_summary=out_of_time_robustness_summary,
|
|
rqs_score=rqs_score,
|
|
wfqs_v2_score=wfqs_v2_score,
|
|
)
|
|
if components is None:
|
|
return None, breakdown, source
|
|
|
|
final_score = (
|
|
components["base_score"]
|
|
* components["rb_gate_factor"]
|
|
* components["oot_gate_factor"]
|
|
* components["activity_factor"]
|
|
)
|
|
resolved_breakdown = {
|
|
"base_score": round(components["base_score"], 1),
|
|
"gate_factor": round(components["rb_gate_factor"], 2),
|
|
**components["rb_gate_breakdown"],
|
|
"oot_gate_factor": round(components["oot_gate_factor"], 2),
|
|
**{
|
|
f"oot_{key}": value
|
|
for key, value in components["oot_gate_breakdown"].items()
|
|
},
|
|
**{
|
|
f"oot_{key}": value
|
|
for key, value in components["oot_quality_breakdown"].items()
|
|
},
|
|
**components["activity_breakdown"],
|
|
}
|
|
if components["oot_quality"] is not None:
|
|
resolved_breakdown["oot_quality"] = round(components["oot_quality"], 1)
|
|
return round(final_score, 1), resolved_breakdown, "v3_deployment+robustness+oot_gate"
|
|
|
|
|
|
def compute_public_sqs_v4(
|
|
train_result: SplitResult | None,
|
|
valid_result: SplitResult | None,
|
|
test_result: SplitResult | None,
|
|
walk_forward_summary: WalkForwardSummary | None = None,
|
|
robustness_matrix_summary: RobustnessMatrixSummary | None = None,
|
|
out_of_time_robustness_summary: RobustnessMatrixSummary | None = None,
|
|
common_window_summary: CommonWindowSummary | None = None,
|
|
rqs_score: float | None = None,
|
|
wfqs_v2_score: float | None = None,
|
|
) -> tuple[float | None, dict[str, float], str | None]:
|
|
"""Return public SQS v4.
|
|
|
|
v4 keeps v3 intact as the deployment/stress backbone, then blends in a
|
|
modest full-cycle capital-growth score when a comparable common-window
|
|
run is available. If no common-window summary exists, v4 falls back to v3.
|
|
"""
|
|
v3_score, v3_breakdown, source = compute_public_sqs_v3(
|
|
train_result,
|
|
valid_result,
|
|
test_result,
|
|
walk_forward_summary=walk_forward_summary,
|
|
robustness_matrix_summary=robustness_matrix_summary,
|
|
out_of_time_robustness_summary=out_of_time_robustness_summary,
|
|
rqs_score=rqs_score,
|
|
wfqs_v2_score=wfqs_v2_score,
|
|
)
|
|
if v3_score is None:
|
|
return None, v3_breakdown, source
|
|
|
|
common_score, common_breakdown = compute_common_window_score(common_window_summary)
|
|
if common_score is None:
|
|
return v3_score, v3_breakdown, "v4_fallback_v3_missing_common_window"
|
|
|
|
weight = _DEFAULT_PUBLIC_COMMON_WINDOW_WEIGHT
|
|
final_score = v3_score * (1.0 - weight) + common_score * weight
|
|
breakdown = {
|
|
"v3_score": round(v3_score, 1),
|
|
"common_window_score": round(common_score, 1),
|
|
"common_window_weight": round(weight, 2),
|
|
**v3_breakdown,
|
|
**common_breakdown,
|
|
}
|
|
return round(final_score, 1), breakdown, "v4_deployment+common_window"
|
|
|
|
|
|
def compute_deployment_score(
|
|
train_result: SplitResult | None,
|
|
valid_result: SplitResult | None,
|
|
test_result: SplitResult | None,
|
|
walk_forward_summary: WalkForwardSummary | None,
|
|
rqs_score: float | None = None,
|
|
wfqs_score: float | None = None,
|
|
weights: DeploymentScoreWeights | None = None,
|
|
) -> tuple[float | None, dict[str, float]]:
|
|
"""Compute a deployment-oriented score that requires walk-forward confirmation."""
|
|
if walk_forward_summary is None:
|
|
return None, {}
|
|
|
|
rqs = rqs_score
|
|
if rqs is None:
|
|
rqs, _ = compute_rqs(train_result, valid_result, test_result)
|
|
wfqs = wfqs_score
|
|
if wfqs is None:
|
|
wfqs, _ = compute_wfqs(walk_forward_summary)
|
|
if rqs is None or wfqs is None:
|
|
return None, {}
|
|
|
|
w = weights or _DEFAULT_DEPLOYMENT_WEIGHTS
|
|
test = walk_forward_summary.test_aggregate
|
|
gap = walk_forward_summary.gap_stats
|
|
|
|
pass_positive = (test.positive_fold_rate_pct or 0.0) >= 70.0
|
|
pass_median = (test.median_return_pct or 0.0) >= 5.0
|
|
pass_worst = (test.worst_return_pct or 0.0) >= -5.0
|
|
pass_gap = (gap.mean_train_test_return_gap_pct or 999.0) <= 35.0
|
|
pass_count = sum([pass_positive, pass_median, pass_worst, pass_gap])
|
|
gate_factor = {
|
|
4: 1.00,
|
|
3: 0.85,
|
|
2: 0.65,
|
|
1: 0.40,
|
|
0: 0.20,
|
|
}[pass_count]
|
|
|
|
deployment = (rqs * w.rqs_quality + wfqs * w.wfqs_quality) * gate_factor
|
|
breakdown = {
|
|
"rqs_quality": round(rqs, 1),
|
|
"wfqs_quality": round(wfqs, 1),
|
|
"gate_factor": round(gate_factor, 2),
|
|
"gate_positive_fold_rate": 1.0 if pass_positive else 0.0,
|
|
"gate_median_return": 1.0 if pass_median else 0.0,
|
|
"gate_worst_return": 1.0 if pass_worst else 0.0,
|
|
"gate_train_test_gap": 1.0 if pass_gap else 0.0,
|
|
}
|
|
return round(deployment, 1), breakdown
|
|
|
|
|
|
def compute_common_window_score(
|
|
common_window_summary: CommonWindowSummary | None,
|
|
) -> tuple[float | None, dict[str, float]]:
|
|
"""Score a continuous full-cycle run for capital-growth / recycling quality."""
|
|
if common_window_summary is None:
|
|
return None, {}
|
|
|
|
metrics = common_window_summary.metrics
|
|
total_return_score = _normalize(metrics.total_return_pct, low=0.0, high=250.0)
|
|
profit_factor_score = _normalize(metrics.profit_factor, low=1.0, high=6.0)
|
|
sharpe_score = _normalize(metrics.sharpe_ratio, low=0.0, high=3.0)
|
|
drawdown_score = _normalize_inverse(metrics.max_drawdown_pct, low=12.0, high=2.5)
|
|
return_on_gross = _safe_ratio(metrics.total_return_pct, metrics.avg_gross_exposure_pct)
|
|
return_on_gross_score = _normalize(return_on_gross, low=1.0, high=8.0)
|
|
capital_velocity = _safe_ratio(metrics.total_return_pct, metrics.days_in_market_pct)
|
|
capital_velocity_score = _normalize(capital_velocity, low=0.4, high=3.0)
|
|
|
|
score = (
|
|
total_return_score * 0.35
|
|
+ profit_factor_score * 0.10
|
|
+ sharpe_score * 0.15
|
|
+ drawdown_score * 0.15
|
|
+ return_on_gross_score * 0.20
|
|
+ capital_velocity_score * 0.05
|
|
)
|
|
if metrics.trade_count < 25:
|
|
score *= 0.85
|
|
|
|
breakdown = {
|
|
"cw_total_return": round(total_return_score, 1),
|
|
"cw_profit_factor": round(profit_factor_score, 1),
|
|
"cw_sharpe": round(sharpe_score, 1),
|
|
"cw_drawdown": round(drawdown_score, 1),
|
|
"cw_return_on_gross": round(return_on_gross_score, 1),
|
|
"cw_capital_velocity": round(capital_velocity_score, 1),
|
|
"cw_trade_count": float(metrics.trade_count),
|
|
}
|
|
return round(score, 1), breakdown
|
|
|
|
|
|
def _get_robustness_horizon_summary(
|
|
summary: RobustnessMatrixSummary | None,
|
|
horizon_days: int,
|
|
) -> dict[str, float | int | None] | None:
|
|
if summary is None:
|
|
return None
|
|
for item in summary.horizon_summaries:
|
|
if item.horizon_days == horizon_days:
|
|
return item.model_dump()
|
|
return None
|
|
|
|
|
|
def compute_robustness_gate(
|
|
robustness_matrix_summary: RobustnessMatrixSummary | None,
|
|
) -> tuple[float, dict[str, float]]:
|
|
"""Return multiplicative gate from the robustness matrix summary."""
|
|
if robustness_matrix_summary is None:
|
|
return 1.0, {}
|
|
|
|
h63 = _get_robustness_horizon_summary(robustness_matrix_summary, 63)
|
|
h252 = _get_robustness_horizon_summary(robustness_matrix_summary, 252)
|
|
pass_positive = (robustness_matrix_summary.overall_positive_window_rate_pct or 0.0) >= 65.0
|
|
pass_63 = ((h63 or {}).get("median_return_pct") or 0.0) >= 3.0
|
|
pass_252 = ((h252 or {}).get("median_return_pct") or 0.0) >= 8.0
|
|
pass_worst = (robustness_matrix_summary.overall_worst_return_pct or -999.0) >= -12.0
|
|
pass_count = sum([pass_positive, pass_63, pass_252, pass_worst])
|
|
gate_factor = {
|
|
4: 1.00,
|
|
3: 0.85,
|
|
2: 0.65,
|
|
1: 0.40,
|
|
0: 0.20,
|
|
}[pass_count]
|
|
breakdown = {
|
|
"rb_gate_positive_rate": 1.0 if pass_positive else 0.0,
|
|
"rb_gate_63d_median": 1.0 if pass_63 else 0.0,
|
|
"rb_gate_252d_median": 1.0 if pass_252 else 0.0,
|
|
"rb_gate_worst_return": 1.0 if pass_worst else 0.0,
|
|
}
|
|
return gate_factor, breakdown
|
|
|
|
|
|
def _compute_oot_positive_rate_63plus(
|
|
summary: RobustnessMatrixSummary,
|
|
) -> float | None:
|
|
"""63d+ horizon weighted-average positive window rate.
|
|
|
|
21d windows penalise catalyst strategies unfairly (0-trade windows
|
|
count as non-positive). 63d+ windows give a fairer survival signal.
|
|
"""
|
|
qualifying = [h for h in summary.horizon_summaries if h.horizon_days >= 63]
|
|
if not qualifying:
|
|
return None
|
|
total_windows = sum(h.window_count for h in qualifying)
|
|
if total_windows == 0:
|
|
return None
|
|
weighted_positive = sum(
|
|
(h.positive_window_rate_pct or 0.0) * h.window_count
|
|
for h in qualifying
|
|
)
|
|
return round(weighted_positive / total_windows, 1)
|
|
|
|
|
|
def compute_oot_robustness_gate(
|
|
robustness_matrix_summary: RobustnessMatrixSummary | None,
|
|
) -> tuple[float, dict[str, float]]:
|
|
"""OOT stress-test robustness gate with relaxed thresholds.
|
|
|
|
Stress-test thresholds (vs main-period):
|
|
- 63d+ positive rate >= 50% (main: overall >= 65%)
|
|
- 63d median >= 0.5% (main: >= 3.0%)
|
|
- 252d median >= 3.0% (main: >= 8.0%)
|
|
- worst return >= -15.0% (main: >= -12.0%)
|
|
"""
|
|
if robustness_matrix_summary is None:
|
|
return 1.0, {}
|
|
|
|
h63 = _get_robustness_horizon_summary(robustness_matrix_summary, 63)
|
|
h252 = _get_robustness_horizon_summary(robustness_matrix_summary, 252)
|
|
positive_rate_63plus = _compute_oot_positive_rate_63plus(robustness_matrix_summary)
|
|
pass_positive = (positive_rate_63plus or 0.0) >= 50.0
|
|
pass_63 = ((h63 or {}).get("median_return_pct") or 0.0) >= 0.5
|
|
pass_252 = ((h252 or {}).get("median_return_pct") or 0.0) >= 3.0
|
|
pass_worst = (robustness_matrix_summary.overall_worst_return_pct or -999.0) >= -15.0
|
|
pass_count = sum([pass_positive, pass_63, pass_252, pass_worst])
|
|
gate_factor = {
|
|
4: 1.00,
|
|
3: 0.85,
|
|
2: 0.65,
|
|
1: 0.40,
|
|
0: 0.20,
|
|
}[pass_count]
|
|
breakdown = {
|
|
"rb_gate_positive_rate": 1.0 if pass_positive else 0.0,
|
|
"rb_gate_63d_median": 1.0 if pass_63 else 0.0,
|
|
"rb_gate_252d_median": 1.0 if pass_252 else 0.0,
|
|
"rb_gate_worst_return": 1.0 if pass_worst else 0.0,
|
|
}
|
|
return gate_factor, breakdown
|
|
|
|
|
|
def compute_robustness_quality(
|
|
robustness_matrix_summary: RobustnessMatrixSummary | None,
|
|
) -> tuple[float | None, dict[str, float]]:
|
|
"""Return a continuous robustness quality score in [0, 100].
|
|
|
|
This complements the coarse pass/fail gate so repaired OOT runs can
|
|
meaningfully reorder strategies instead of collapsing into the same gate.
|
|
"""
|
|
if robustness_matrix_summary is None:
|
|
return None, {}
|
|
|
|
h63 = _get_robustness_horizon_summary(robustness_matrix_summary, 63)
|
|
h252 = _get_robustness_horizon_summary(robustness_matrix_summary, 252)
|
|
positive_rate_score = _normalize(
|
|
robustness_matrix_summary.overall_positive_window_rate_pct,
|
|
low=30.0,
|
|
high=80.0,
|
|
)
|
|
worst_return_score = _normalize(
|
|
robustness_matrix_summary.overall_worst_return_pct,
|
|
low=-25.0,
|
|
high=0.0,
|
|
)
|
|
h63_median_score = _normalize(
|
|
(h63 or {}).get("median_return_pct"),
|
|
low=-5.0,
|
|
high=5.0,
|
|
)
|
|
h252_median_score = _normalize(
|
|
(h252 or {}).get("median_return_pct"),
|
|
low=-10.0,
|
|
high=15.0,
|
|
)
|
|
quality = (
|
|
positive_rate_score * 0.30
|
|
+ h63_median_score * 0.25
|
|
+ h252_median_score * 0.30
|
|
+ worst_return_score * 0.15
|
|
)
|
|
breakdown = {
|
|
"rb_quality_positive_rate": round(positive_rate_score, 1),
|
|
"rb_quality_63d_median": round(h63_median_score, 1),
|
|
"rb_quality_252d_median": round(h252_median_score, 1),
|
|
"rb_quality_worst_return": round(worst_return_score, 1),
|
|
}
|
|
return round(quality, 1), breakdown
|
|
|
|
|
|
def compute_oot_robustness_quality(
|
|
robustness_matrix_summary: RobustnessMatrixSummary | None,
|
|
) -> tuple[float | None, dict[str, float]]:
|
|
"""OOT stress-test continuous quality score [0, 100].
|
|
|
|
Normalisation ranges adjusted for stress-test periods (e.g. COVID).
|
|
Uses 63d+ positive rate instead of overall (excludes 21d noise).
|
|
"""
|
|
if robustness_matrix_summary is None:
|
|
return None, {}
|
|
|
|
h63 = _get_robustness_horizon_summary(robustness_matrix_summary, 63)
|
|
h252 = _get_robustness_horizon_summary(robustness_matrix_summary, 252)
|
|
positive_rate_63plus = _compute_oot_positive_rate_63plus(robustness_matrix_summary)
|
|
positive_rate_score = _normalize(
|
|
positive_rate_63plus,
|
|
low=20.0,
|
|
high=70.0,
|
|
)
|
|
worst_return_score = _normalize(
|
|
robustness_matrix_summary.overall_worst_return_pct,
|
|
low=-30.0,
|
|
high=-5.0,
|
|
)
|
|
h63_median_score = _normalize(
|
|
(h63 or {}).get("median_return_pct"),
|
|
low=-8.0,
|
|
high=3.0,
|
|
)
|
|
h252_median_score = _normalize(
|
|
(h252 or {}).get("median_return_pct"),
|
|
low=-15.0,
|
|
high=10.0,
|
|
)
|
|
quality = (
|
|
positive_rate_score * 0.30
|
|
+ h63_median_score * 0.25
|
|
+ h252_median_score * 0.30
|
|
+ worst_return_score * 0.15
|
|
)
|
|
breakdown = {
|
|
"rb_quality_positive_rate": round(positive_rate_score, 1),
|
|
"rb_quality_63d_median": round(h63_median_score, 1),
|
|
"rb_quality_252d_median": round(h252_median_score, 1),
|
|
"rb_quality_worst_return": round(worst_return_score, 1),
|
|
}
|
|
return round(quality, 1), breakdown
|
|
|
|
|
|
def compute_config_delta(
|
|
current: dict[str, Any],
|
|
baseline: dict[str, Any],
|
|
baseline_name: str,
|
|
) -> ConfigDelta:
|
|
"""Compute a flat diff between two config dicts (one level deep)."""
|
|
changes: dict[str, str] = {}
|
|
_diff_recursive(baseline, current, prefix="", changes=changes)
|
|
return ConfigDelta(base_experiment=baseline_name, changes=changes)
|
|
|
|
|
|
def _diff_recursive(
|
|
old: dict[str, Any],
|
|
new: dict[str, Any],
|
|
prefix: str,
|
|
changes: dict[str, str],
|
|
) -> None:
|
|
all_keys = set(old.keys()) | set(new.keys())
|
|
for key in sorted(all_keys):
|
|
full_key = f"{prefix}{key}" if not prefix else f"{prefix}.{key}"
|
|
old_val = old.get(key)
|
|
new_val = new.get(key)
|
|
if isinstance(old_val, dict) and isinstance(new_val, dict):
|
|
_diff_recursive(old_val, new_val, full_key, changes)
|
|
elif old_val != new_val:
|
|
changes[full_key] = f"{old_val} \u2192 {new_val}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Journal I/O
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def get_next_entry_id(journal_path: Path) -> str:
|
|
"""Return next entry ID like 'IMP-0001'."""
|
|
entries = load_journal(journal_path)
|
|
return f"IMP-{len(entries) + 1:04d}"
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def journal_lock(journal_path: Path) -> Iterator[None]:
|
|
"""Serialize journal mutations across concurrent record commands."""
|
|
journal_path.parent.mkdir(parents=True, exist_ok=True)
|
|
lock_path = journal_path.with_suffix(f"{journal_path.suffix}.lock")
|
|
with lock_path.open("a+") as lock_file:
|
|
fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
|
|
try:
|
|
yield
|
|
finally:
|
|
fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
|
|
|
|
|
|
def append_journal_entry(journal_path: Path, entry: JournalEntry) -> None:
|
|
"""Append a single JournalEntry as one JSON line."""
|
|
journal_path.parent.mkdir(parents=True, exist_ok=True)
|
|
with open(journal_path, "a") as f:
|
|
f.write(entry.model_dump_json() + "\n")
|
|
logger.info("journal_entry_appended", entry_id=entry.entry_id, experiment=entry.experiment_name)
|
|
|
|
|
|
def replace_journal_entry(journal_path: Path, entry: JournalEntry) -> None:
|
|
"""Replace an existing journal entry by entry_id."""
|
|
entries = load_journal(journal_path)
|
|
replaced = False
|
|
updated_entries: list[JournalEntry] = []
|
|
for current in entries:
|
|
if current.entry_id == entry.entry_id:
|
|
updated_entries.append(entry)
|
|
replaced = True
|
|
else:
|
|
updated_entries.append(current)
|
|
if not replaced:
|
|
raise ValueError(f"Journal entry not found: {entry.entry_id}")
|
|
journal_path.write_text("".join(e.model_dump_json() + "\n" for e in updated_entries))
|
|
logger.info("journal_entry_replaced", entry_id=entry.entry_id, experiment=entry.experiment_name)
|
|
|
|
|
|
def load_journal(journal_path: Path) -> list[JournalEntry]:
|
|
"""Load all journal entries from JSONL file."""
|
|
if not journal_path.exists():
|
|
return []
|
|
entries: list[JournalEntry] = []
|
|
for line in journal_path.read_text().strip().splitlines():
|
|
line = line.strip()
|
|
if line:
|
|
entries.append(JournalEntry.model_validate_json(line))
|
|
return entries
|
|
|
|
|
|
def refresh_public_scores(
|
|
journal_path: Path,
|
|
selector: Callable[[JournalEntry], bool] | None = None,
|
|
) -> int:
|
|
"""Recompute stored public scores and score diagnostics for journal entries."""
|
|
entries = load_journal(journal_path)
|
|
updated_entries: list[JournalEntry] = []
|
|
updated_count = 0
|
|
|
|
for entry in entries:
|
|
if selector is not None and not selector(entry):
|
|
updated_entries.append(entry)
|
|
continue
|
|
|
|
test_result = _hydrate_split_result(entry.results.get("test"))
|
|
valid_result = _hydrate_split_result(entry.results.get("valid"))
|
|
train_result = _hydrate_split_result(entry.results.get("train"))
|
|
test_metrics = _metrics_from_split_result(test_result)
|
|
|
|
sqs_v2_score = None
|
|
sqs_v2_breakdown: dict[str, float] = {}
|
|
if test_metrics is not None:
|
|
sqs_v2_score, sqs_v2_breakdown = compute_sqs_v2(test_metrics)
|
|
|
|
rqs_score, rqs_breakdown = compute_rqs(train_result, valid_result, test_result)
|
|
wfqs_score, wfqs_breakdown = compute_wfqs(entry.walk_forward_summary)
|
|
wfqs_v2_score, wfqs_v2_breakdown = compute_wfqs_v2(entry.walk_forward_summary)
|
|
deployment_score, deployment_breakdown = compute_deployment_score(
|
|
train_result,
|
|
valid_result,
|
|
test_result,
|
|
entry.walk_forward_summary,
|
|
rqs_score=rqs_score,
|
|
wfqs_score=wfqs_score,
|
|
)
|
|
common_window_score, common_window_breakdown = compute_common_window_score(
|
|
entry.common_window_summary
|
|
)
|
|
sqs_v3_score, sqs_v3_breakdown, _ = compute_public_sqs_v3(
|
|
train_result,
|
|
valid_result,
|
|
test_result,
|
|
walk_forward_summary=entry.walk_forward_summary,
|
|
robustness_matrix_summary=entry.robustness_matrix_summary,
|
|
out_of_time_robustness_summary=entry.out_of_time_robustness_summary,
|
|
rqs_score=rqs_score,
|
|
wfqs_v2_score=wfqs_v2_score,
|
|
)
|
|
sqs_score, sqs_breakdown, _ = compute_public_sqs(
|
|
train_result,
|
|
valid_result,
|
|
test_result,
|
|
walk_forward_summary=entry.walk_forward_summary,
|
|
robustness_matrix_summary=entry.robustness_matrix_summary,
|
|
out_of_time_robustness_summary=entry.out_of_time_robustness_summary,
|
|
common_window_summary=entry.common_window_summary,
|
|
rqs_score=rqs_score,
|
|
wfqs_score=wfqs_v2_score,
|
|
)
|
|
stress_sqs_score, stress_sqs_breakdown, _ = compute_public_sqs_v2(
|
|
train_result,
|
|
valid_result,
|
|
test_result,
|
|
walk_forward_summary=entry.walk_forward_summary,
|
|
robustness_matrix_summary=entry.robustness_matrix_summary,
|
|
out_of_time_robustness_summary=entry.out_of_time_robustness_summary,
|
|
rqs_score=rqs_score,
|
|
wfqs_v2_score=wfqs_v2_score,
|
|
)
|
|
|
|
refreshed = entry.model_copy(
|
|
update={
|
|
"sqs_score": sqs_score,
|
|
"sqs_breakdown": sqs_breakdown,
|
|
"sqs_v3_score": sqs_v3_score,
|
|
"sqs_v3_breakdown": sqs_v3_breakdown,
|
|
"stress_sqs_score": stress_sqs_score,
|
|
"stress_sqs_breakdown": stress_sqs_breakdown,
|
|
"sqs_v2_score": sqs_v2_score,
|
|
"sqs_v2_breakdown": sqs_v2_breakdown,
|
|
"rqs_score": rqs_score,
|
|
"rqs_breakdown": rqs_breakdown,
|
|
"wfqs_score": wfqs_score,
|
|
"wfqs_breakdown": wfqs_breakdown,
|
|
"wfqs_v2_score": wfqs_v2_score,
|
|
"wfqs_v2_breakdown": wfqs_v2_breakdown,
|
|
"deployment_score": deployment_score,
|
|
"deployment_breakdown": deployment_breakdown,
|
|
"common_window_score": common_window_score,
|
|
"common_window_breakdown": common_window_breakdown,
|
|
}
|
|
)
|
|
if refreshed.model_dump() != entry.model_dump():
|
|
updated_count += 1
|
|
updated_entries.append(refreshed)
|
|
|
|
if updated_count:
|
|
journal_path.write_text("".join(item.model_dump_json() + "\n" for item in updated_entries))
|
|
logger.info("journal_public_scores_refreshed", updated_count=updated_count)
|
|
|
|
return updated_count
|
|
|
|
|
|
def _resolve_journal_target(entries: list[JournalEntry], target: str) -> JournalEntry:
|
|
target_upper = target.upper()
|
|
by_id = [entry for entry in entries if entry.entry_id.upper() == target_upper]
|
|
if by_id:
|
|
return by_id[0]
|
|
|
|
exact_name = [entry for entry in entries if entry.experiment_name == target]
|
|
if len(exact_name) == 1:
|
|
return exact_name[0]
|
|
if len(exact_name) > 1:
|
|
return sorted(exact_name, key=lambda e: e.timestamp)[-1]
|
|
|
|
partial = [entry for entry in entries if target.lower() in entry.experiment_name.lower()]
|
|
if len(partial) != 1:
|
|
raise ValueError(f"Unable to uniquely match journal entry: {target}")
|
|
return partial[0]
|
|
|
|
|
|
def attach_walk_forward_summary(
|
|
journal_path: Path,
|
|
target: str,
|
|
summary: WalkForwardSummary,
|
|
) -> JournalEntry:
|
|
"""Attach walk-forward validation summary to an existing journal entry."""
|
|
entries = load_journal(journal_path)
|
|
selected = _resolve_journal_target(entries, target)
|
|
|
|
wfqs_score, wfqs_breakdown = compute_wfqs(summary)
|
|
wfqs_v2_score, wfqs_v2_breakdown = compute_wfqs_v2(summary)
|
|
deployment_score, deployment_breakdown = compute_deployment_score(
|
|
selected.results.get("train"),
|
|
selected.results.get("valid"),
|
|
selected.results.get("test"),
|
|
summary,
|
|
)
|
|
updated = selected.model_copy(
|
|
update={
|
|
"walk_forward_summary": summary,
|
|
"wfqs_score": wfqs_score,
|
|
"wfqs_breakdown": wfqs_breakdown,
|
|
"wfqs_v2_score": wfqs_v2_score,
|
|
"wfqs_v2_breakdown": wfqs_v2_breakdown,
|
|
"deployment_score": deployment_score,
|
|
"deployment_breakdown": deployment_breakdown,
|
|
}
|
|
)
|
|
replace_journal_entry(journal_path, updated)
|
|
return updated
|
|
|
|
|
|
def attach_robustness_summary(
|
|
journal_path: Path,
|
|
target: str,
|
|
summary: RobustnessMatrixSummary,
|
|
) -> JournalEntry:
|
|
"""Attach robustness matrix summary to an existing journal entry."""
|
|
entries = load_journal(journal_path)
|
|
selected = _resolve_journal_target(entries, target)
|
|
|
|
updated = selected.model_copy(
|
|
update={
|
|
"robustness_matrix_summary": summary,
|
|
}
|
|
)
|
|
replace_journal_entry(journal_path, updated)
|
|
return updated
|
|
|
|
|
|
def attach_out_of_time_robustness_summary(
|
|
journal_path: Path,
|
|
target: str,
|
|
summary: RobustnessMatrixSummary,
|
|
) -> JournalEntry:
|
|
"""Attach out-of-time robustness summary to an existing journal entry."""
|
|
entries = load_journal(journal_path)
|
|
selected = _resolve_journal_target(entries, target)
|
|
|
|
updated = selected.model_copy(
|
|
update={
|
|
"out_of_time_robustness_summary": summary,
|
|
}
|
|
)
|
|
replace_journal_entry(journal_path, updated)
|
|
return updated
|
|
|
|
|
|
def attach_common_window_summary(
|
|
journal_path: Path,
|
|
target: str,
|
|
summary: CommonWindowSummary,
|
|
) -> JournalEntry:
|
|
"""Attach common-window continuous-run summary to an existing journal entry."""
|
|
entries = load_journal(journal_path)
|
|
selected = _resolve_journal_target(entries, target)
|
|
|
|
common_window_score, common_window_breakdown = compute_common_window_score(summary)
|
|
updated = selected.model_copy(
|
|
update={
|
|
"common_window_summary": summary,
|
|
"common_window_score": common_window_score,
|
|
"common_window_breakdown": common_window_breakdown,
|
|
}
|
|
)
|
|
replace_journal_entry(journal_path, updated)
|
|
return updated
|
|
|
|
|
|
@functools.lru_cache(maxsize=1024)
|
|
def _load_manifest_json(experiment_name: str) -> dict[str, Any] | None:
|
|
path = _EXPERIMENTS_DIR / f"{experiment_name}.json"
|
|
if not path.exists():
|
|
return None
|
|
with path.open("r", encoding="utf-8") as fh:
|
|
return json.load(fh)
|
|
|
|
|
|
def _manifest_has_exact_pocket_structure(experiment_name: str) -> bool:
|
|
manifest = _load_manifest_json(experiment_name)
|
|
if manifest is None:
|
|
return False
|
|
for engine in manifest.get("strategy_engines", []):
|
|
engine_id = str(engine.get("engine_id", "")).lower()
|
|
if engine.get("enabled", True) and "exact" in engine_id:
|
|
return True
|
|
return False
|
|
|
|
|
|
def _manifest_has_named_micro_structure(experiment_name: str) -> bool:
|
|
manifest = _load_manifest_json(experiment_name)
|
|
if manifest is None:
|
|
return False
|
|
for engine in manifest.get("strategy_engines", []):
|
|
engine_id = str(engine.get("engine_id", "")).lower()
|
|
if any(token in engine_id for token in _NAMED_MICRO_ENGINE_TOKENS):
|
|
return True
|
|
return False
|
|
|
|
|
|
def classify_strategy_family(experiment_name: str, tags: list[str] | None = None) -> str:
|
|
"""Classify an experiment into a coarse strategy family for leaderboard filtering."""
|
|
lowered_name = experiment_name.lower()
|
|
lowered_tags = {tag.lower() for tag in tags or []}
|
|
|
|
if lowered_name.startswith("return_book_overlay") or "overlay" in lowered_tags:
|
|
return "overlay"
|
|
if _manifest_has_exact_pocket_structure(experiment_name):
|
|
return "exact_pocket_return_max_long"
|
|
if _manifest_has_named_micro_structure(experiment_name):
|
|
return "named_micro_return_max_long"
|
|
if (
|
|
"exact" in lowered_name
|
|
or "combo" in lowered_name
|
|
or "exact" in lowered_tags
|
|
or "combo" in lowered_tags
|
|
):
|
|
return "exact_pocket_return_max_long"
|
|
if lowered_name.startswith("return_max_long") and "_bp" in lowered_name:
|
|
return "leveraged_return_max_long"
|
|
if lowered_name.startswith("return_max_long"):
|
|
return "return_max_long"
|
|
if "short_core" in lowered_name or "short_core" in lowered_tags:
|
|
return "short_core"
|
|
if lowered_name.startswith("pead_") or "pead" in lowered_tags:
|
|
return "legacy_pead"
|
|
return "other"
|
|
|
|
|
|
def is_retired_strategy_family(strategy_family: str) -> bool:
|
|
"""Return True when the strategy family is retired from the default workflow."""
|
|
return strategy_family in _RETIRED_STRATEGY_FAMILIES
|
|
|
|
|
|
def filter_registry_entries(
|
|
entries: list[RegistryEntry],
|
|
*,
|
|
include_retired: bool = False,
|
|
) -> list[RegistryEntry]:
|
|
"""Filter registry entries for the default surfaced leaderboard."""
|
|
filtered = list(entries) if include_retired else [entry for entry in entries if not entry.is_retired]
|
|
return [
|
|
entry
|
|
for entry in filtered
|
|
if (
|
|
entry.sqs_score is not None
|
|
and entry.strategy_family != "overlay"
|
|
and entry.trade_count > 0
|
|
and entry.valid_trade_count > 0
|
|
)
|
|
]
|
|
|
|
|
|
def _is_retired_journal_entry(entry: JournalEntry) -> bool:
|
|
return is_retired_strategy_family(classify_strategy_family(entry.experiment_name, entry.tags))
|
|
|
|
|
|
def _is_complete_journal_entry(entry: JournalEntry) -> bool:
|
|
test_result = entry.results.get("test")
|
|
valid_result = entry.results.get("valid")
|
|
return (
|
|
test_result is not None
|
|
and valid_result is not None
|
|
and test_result.trade_count > 0
|
|
and valid_result.trade_count > 0
|
|
)
|
|
|
|
|
|
def _load_optional_walk_forward_summary(
|
|
experiment_name: str,
|
|
runs_dir: Path | None = None,
|
|
) -> WalkForwardSummary | None:
|
|
runs_root = runs_dir or Path("runs")
|
|
summary_path = runs_root / f"{experiment_name}_wfv" / "walk_forward" / "walk_forward_summary.json"
|
|
if not summary_path.exists():
|
|
return None
|
|
return WalkForwardSummary.model_validate_json(summary_path.read_text())
|
|
|
|
|
|
def _load_optional_robustness_summary(
|
|
experiment_name: str,
|
|
runs_dir: Path | None = None,
|
|
) -> RobustnessMatrixSummary | None:
|
|
runs_root = runs_dir or Path("runs")
|
|
summary_path = runs_root / f"{experiment_name}_rm" / "robustness_matrix" / "robustness_matrix_summary.json"
|
|
if not summary_path.exists():
|
|
return None
|
|
return RobustnessMatrixSummary.model_validate_json(summary_path.read_text())
|
|
|
|
|
|
def _load_optional_out_of_time_robustness_summary(
|
|
experiment_name: str,
|
|
runs_dir: Path | None = None,
|
|
) -> RobustnessMatrixSummary | None:
|
|
runs_root = runs_dir or Path("runs")
|
|
summary_path = runs_root / f"{experiment_name}_oot_rm" / "robustness_matrix" / "robustness_matrix_summary.json"
|
|
if not summary_path.exists():
|
|
return None
|
|
return RobustnessMatrixSummary.model_validate_json(summary_path.read_text())
|
|
|
|
|
|
def _build_manifest_index(
|
|
runs_dir: Path,
|
|
) -> dict[str, list[tuple[Path, dict, dict]]]:
|
|
"""Build experiment_name -> [(run_dir, manifest_data, metadata)] index in one rglob pass."""
|
|
index: dict[str, list[tuple[Path, dict, dict]]] = {}
|
|
for manifest_file in sorted(runs_dir.rglob("manifest.json")):
|
|
run_path = manifest_file.parent
|
|
try:
|
|
manifest_data = json.loads(manifest_file.read_text())
|
|
except Exception:
|
|
continue
|
|
exp_name = manifest_data.get("experiment_name")
|
|
if not exp_name:
|
|
continue
|
|
metadata_file = run_path / "metadata.json"
|
|
try:
|
|
metadata = json.loads(metadata_file.read_text()) if metadata_file.exists() else {}
|
|
except Exception:
|
|
metadata = {}
|
|
index.setdefault(exp_name, []).append((run_path, manifest_data, metadata))
|
|
return index
|
|
|
|
|
|
def _scan_runs_from_index(
|
|
run_entries: list[tuple[Path, dict, dict]],
|
|
) -> dict[str, tuple[str, "MetricsBundle"]]:
|
|
"""Resolve split runs from pre-indexed manifest entries (no rglob)."""
|
|
from libs.backtest.domain import MetricsBundle
|
|
|
|
matches: list[tuple[Path, str, str, str, str | None]] = []
|
|
for run_path, _manifest_data, metadata in run_entries:
|
|
run_id = metadata.get("run_id", run_path.name)
|
|
started_at = str(metadata.get("started_at") or metadata.get("finished_at") or "")
|
|
split_name = metadata.get("split_name")
|
|
parts = run_path.name.split("_")
|
|
config_hash = parts[-1] if len(parts) >= 2 else ""
|
|
matches.append((run_path, run_id, config_hash, started_at, split_name))
|
|
|
|
if not matches:
|
|
return {}
|
|
|
|
results: dict[str, tuple[str, MetricsBundle]] = {}
|
|
has_split_names = any(sn is not None for _, _, _, _, sn in matches)
|
|
canonical_splits = {"train", "valid", "test"}
|
|
|
|
if has_split_names:
|
|
canonical_matches = [m for m in matches if m[4] in canonical_splits]
|
|
if not canonical_matches:
|
|
return {}
|
|
latest_by_split: dict[str, tuple[str, Path, str]] = {}
|
|
for run_path, run_id, _, started_at, split_name in canonical_matches:
|
|
split = split_name or "unknown"
|
|
current = latest_by_split.get(split)
|
|
if current is None or started_at >= current[0]:
|
|
latest_by_split[split] = (started_at, run_path, run_id)
|
|
for split, (_, run_path, run_id) in latest_by_split.items():
|
|
metrics_file = run_path / "metrics" / "metrics_summary.json"
|
|
if metrics_file.exists():
|
|
metrics = MetricsBundle.model_validate_json(metrics_file.read_text())
|
|
results[split] = (run_id, metrics)
|
|
else:
|
|
from collections import defaultdict
|
|
groups: dict[str, list[tuple[str, Path, str]]] = defaultdict(list)
|
|
for run_path, run_id, config_hash, started_at, _ in matches:
|
|
groups[config_hash].append((started_at, run_path, run_id))
|
|
best_hash = max(groups, key=lambda h: max(t[0] for t in groups[h]))
|
|
sorted_runs = sorted(groups[best_hash], key=lambda t: t[0])
|
|
split_order = ["train", "valid", "test"]
|
|
for i, (_, run_path, run_id) in enumerate(sorted_runs[-3:]):
|
|
split = split_order[i] if i < 3 else f"extra_{i}"
|
|
metrics_file = run_path / "metrics" / "metrics_summary.json"
|
|
if metrics_file.exists():
|
|
metrics = MetricsBundle.model_validate_json(metrics_file.read_text())
|
|
results[split] = (run_id, metrics)
|
|
|
|
return results
|
|
|
|
|
|
def sync_official_manifests(
|
|
journal_path: Path,
|
|
runs_dir: Path,
|
|
configs_dir: Path | None = None,
|
|
) -> list[JournalEntry]:
|
|
"""Append official manifests with complete runs that are missing from the journal."""
|
|
configs_root = configs_dir or _EXPERIMENTS_DIR
|
|
existing_entries = load_journal(journal_path)
|
|
existing_names = {entry.experiment_name for entry in existing_entries}
|
|
next_index = len(existing_entries) + 1
|
|
synced_entries: list[JournalEntry] = []
|
|
|
|
# Build manifest index once (single rglob) instead of per-experiment
|
|
manifest_index = _build_manifest_index(runs_dir)
|
|
|
|
for manifest_path in sorted(configs_root.glob("*.json")):
|
|
manifest_payload = json.loads(manifest_path.read_text())
|
|
experiment_name = str(manifest_payload.get("experiment_name") or manifest_path.stem)
|
|
if experiment_name in existing_names:
|
|
continue
|
|
|
|
run_entries = manifest_index.get(experiment_name)
|
|
if not run_entries:
|
|
continue
|
|
split_runs = _scan_runs_from_index(run_entries)
|
|
if not {"train", "valid", "test"}.issubset(split_runs):
|
|
continue
|
|
|
|
results: dict[str, SplitResult] = {}
|
|
for split_name in ("train", "valid", "test"):
|
|
run_id, metrics = split_runs[split_name]
|
|
results[split_name] = build_split_result(split_name, run_id, metrics)
|
|
|
|
test_metrics = _metrics_from_split_result(results.get("test"))
|
|
sqs_score = None
|
|
sqs_breakdown: dict[str, float] = {}
|
|
sqs_v2_score = None
|
|
sqs_v2_breakdown: dict[str, float] = {}
|
|
if test_metrics is not None:
|
|
sqs_score, sqs_breakdown = compute_sqs(test_metrics)
|
|
sqs_v2_score, sqs_v2_breakdown = compute_sqs_v2(test_metrics)
|
|
|
|
promotion_score, promotion_breakdown = compute_promotion_score(
|
|
results.get("test"),
|
|
results.get("valid"),
|
|
)
|
|
unified_score, unified_breakdown = compute_unified_score(
|
|
results.get("test"),
|
|
results.get("valid"),
|
|
)
|
|
rqs_score, rqs_breakdown = compute_rqs(
|
|
results.get("train"),
|
|
results.get("valid"),
|
|
results.get("test"),
|
|
)
|
|
walk_forward_summary = _load_optional_walk_forward_summary(experiment_name, runs_dir)
|
|
wfqs_score, wfqs_breakdown = compute_wfqs(walk_forward_summary)
|
|
wfqs_v2_score, wfqs_v2_breakdown = compute_wfqs_v2(walk_forward_summary)
|
|
deployment_score, deployment_breakdown = compute_deployment_score(
|
|
results.get("train"),
|
|
results.get("valid"),
|
|
results.get("test"),
|
|
walk_forward_summary,
|
|
rqs_score=rqs_score,
|
|
wfqs_score=wfqs_score,
|
|
)
|
|
robustness_matrix_summary = _load_optional_robustness_summary(experiment_name, runs_dir)
|
|
out_of_time_robustness_summary = _load_optional_out_of_time_robustness_summary(experiment_name, runs_dir)
|
|
public_sqs, public_breakdown, _ = compute_public_sqs(
|
|
results.get("train"),
|
|
results.get("valid"),
|
|
results.get("test"),
|
|
walk_forward_summary=walk_forward_summary,
|
|
robustness_matrix_summary=robustness_matrix_summary,
|
|
out_of_time_robustness_summary=out_of_time_robustness_summary,
|
|
rqs_score=rqs_score,
|
|
wfqs_score=wfqs_v2_score,
|
|
)
|
|
public_sqs_v3, public_breakdown_v3, _ = compute_public_sqs_v3(
|
|
results.get("train"),
|
|
results.get("valid"),
|
|
results.get("test"),
|
|
walk_forward_summary=walk_forward_summary,
|
|
robustness_matrix_summary=robustness_matrix_summary,
|
|
out_of_time_robustness_summary=out_of_time_robustness_summary,
|
|
rqs_score=rqs_score,
|
|
wfqs_v2_score=wfqs_v2_score,
|
|
)
|
|
stress_sqs, stress_breakdown, _ = compute_public_sqs_v2(
|
|
results.get("train"),
|
|
results.get("valid"),
|
|
results.get("test"),
|
|
walk_forward_summary=walk_forward_summary,
|
|
robustness_matrix_summary=robustness_matrix_summary,
|
|
out_of_time_robustness_summary=out_of_time_robustness_summary,
|
|
rqs_score=rqs_score,
|
|
wfqs_v2_score=wfqs_v2_score,
|
|
)
|
|
if public_sqs is not None:
|
|
sqs_score = public_sqs
|
|
sqs_breakdown = public_breakdown
|
|
|
|
timestamp = utc_now().isoformat()
|
|
entry = JournalEntry(
|
|
entry_id=f"IMP-{next_index:04d}",
|
|
timestamp=timestamp,
|
|
experiment_name=experiment_name,
|
|
hypothesis=str(manifest_payload.get("description") or manifest_payload.get("notes") or ""),
|
|
results=results,
|
|
walk_forward_summary=walk_forward_summary,
|
|
robustness_matrix_summary=robustness_matrix_summary,
|
|
out_of_time_robustness_summary=out_of_time_robustness_summary,
|
|
sqs_score=sqs_score,
|
|
sqs_breakdown=sqs_breakdown,
|
|
sqs_v3_score=public_sqs_v3,
|
|
sqs_v3_breakdown=public_breakdown_v3,
|
|
stress_sqs_score=stress_sqs,
|
|
stress_sqs_breakdown=stress_breakdown,
|
|
sqs_v2_score=sqs_v2_score,
|
|
sqs_v2_breakdown=sqs_v2_breakdown,
|
|
promotion_score=promotion_score,
|
|
promotion_breakdown=promotion_breakdown,
|
|
unified_score=unified_score,
|
|
unified_breakdown=unified_breakdown,
|
|
rqs_score=rqs_score,
|
|
rqs_breakdown=rqs_breakdown,
|
|
wfqs_score=wfqs_score,
|
|
wfqs_breakdown=wfqs_breakdown,
|
|
wfqs_v2_score=wfqs_v2_score,
|
|
wfqs_v2_breakdown=wfqs_v2_breakdown,
|
|
deployment_score=deployment_score,
|
|
deployment_breakdown=deployment_breakdown,
|
|
verdict="unknown",
|
|
verdict_reasoning="Auto-synced from official manifest.",
|
|
next_direction="",
|
|
tags=list(manifest_payload.get("tags") or []),
|
|
)
|
|
append_journal_entry(journal_path, entry)
|
|
synced_entries.append(entry)
|
|
existing_names.add(experiment_name)
|
|
next_index += 1
|
|
|
|
return synced_entries
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Registry / Leaderboard
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def rebuild_registry(
|
|
journal_path: Path,
|
|
registry_path: Path,
|
|
leaderboard_path: Path,
|
|
) -> ExperimentRegistry:
|
|
"""Rebuild experiment_registry.json and LEADERBOARD.md from journal."""
|
|
entries = load_journal(journal_path)
|
|
|
|
registry_entries: list[RegistryEntry] = []
|
|
for je in entries:
|
|
strategy_family = classify_strategy_family(je.experiment_name, je.tags)
|
|
is_retired = is_retired_strategy_family(strategy_family)
|
|
if strategy_family == "overlay":
|
|
continue
|
|
|
|
test_result = _hydrate_split_result(je.results.get("test"))
|
|
valid_result = _hydrate_split_result(je.results.get("valid"))
|
|
train_result = _hydrate_split_result(je.results.get("train"))
|
|
computed_sqs_v2 = None
|
|
computed_rqs = None
|
|
computed_rqs_breakdown: dict[str, float] = {}
|
|
computed_wfqs = None
|
|
computed_wfqs_breakdown: dict[str, float] = {}
|
|
computed_deployment = None
|
|
computed_deployment_breakdown: dict[str, float] = {}
|
|
test_metrics = _metrics_from_split_result(test_result)
|
|
if test_metrics is not None:
|
|
computed_sqs_v2, _ = compute_sqs_v2(test_metrics)
|
|
computed_promotion_score, _ = compute_promotion_score(test_result, valid_result)
|
|
computed_unified_score, _ = compute_unified_score(test_result, valid_result)
|
|
computed_rqs, computed_rqs_breakdown = compute_rqs(
|
|
train_result,
|
|
valid_result,
|
|
test_result,
|
|
)
|
|
computed_wfqs, computed_wfqs_breakdown = compute_wfqs(je.walk_forward_summary)
|
|
computed_wfqs_v2, _ = compute_wfqs_v2(je.walk_forward_summary)
|
|
computed_deployment, computed_deployment_breakdown = compute_deployment_score(
|
|
train_result,
|
|
valid_result,
|
|
test_result,
|
|
je.walk_forward_summary,
|
|
rqs_score=computed_rqs,
|
|
wfqs_score=computed_wfqs,
|
|
)
|
|
computed_common_window, computed_common_window_breakdown = compute_common_window_score(
|
|
je.common_window_summary
|
|
)
|
|
computed_public_sqs_v3, _, _ = compute_public_sqs_v3(
|
|
train_result,
|
|
valid_result,
|
|
test_result,
|
|
walk_forward_summary=je.walk_forward_summary,
|
|
robustness_matrix_summary=je.robustness_matrix_summary,
|
|
out_of_time_robustness_summary=je.out_of_time_robustness_summary,
|
|
rqs_score=computed_rqs,
|
|
wfqs_v2_score=computed_wfqs_v2,
|
|
)
|
|
computed_public_sqs, computed_public_breakdown, _ = compute_public_sqs(
|
|
train_result,
|
|
valid_result,
|
|
test_result,
|
|
walk_forward_summary=je.walk_forward_summary,
|
|
robustness_matrix_summary=je.robustness_matrix_summary,
|
|
out_of_time_robustness_summary=je.out_of_time_robustness_summary,
|
|
common_window_summary=je.common_window_summary,
|
|
rqs_score=computed_rqs,
|
|
wfqs_score=computed_wfqs_v2,
|
|
)
|
|
computed_stress_sqs, _, _ = compute_public_sqs_v2(
|
|
train_result,
|
|
valid_result,
|
|
test_result,
|
|
walk_forward_summary=je.walk_forward_summary,
|
|
robustness_matrix_summary=je.robustness_matrix_summary,
|
|
out_of_time_robustness_summary=je.out_of_time_robustness_summary,
|
|
rqs_score=computed_rqs,
|
|
wfqs_v2_score=computed_wfqs_v2,
|
|
)
|
|
canonical_sqs = computed_public_sqs
|
|
registry_entries.append(
|
|
RegistryEntry(
|
|
entry_id=je.entry_id,
|
|
experiment_name=je.experiment_name,
|
|
strategy_family=strategy_family,
|
|
is_retired=is_retired,
|
|
sqs_score=canonical_sqs,
|
|
sqs_v3_score=computed_public_sqs_v3,
|
|
stress_sqs_score=computed_stress_sqs,
|
|
sqs_v2_score=je.sqs_v2_score if je.sqs_v2_score is not None else computed_sqs_v2,
|
|
promotion_score=(
|
|
je.promotion_score if je.promotion_score is not None else computed_promotion_score
|
|
),
|
|
unified_score=(
|
|
je.unified_score if je.unified_score is not None else computed_unified_score
|
|
),
|
|
rqs_score=computed_rqs if computed_rqs is not None else je.rqs_score,
|
|
wfqs_score=computed_wfqs if computed_wfqs is not None else je.wfqs_score,
|
|
wfqs_v2_score=computed_wfqs_v2 if computed_wfqs_v2 is not None else je.wfqs_v2_score,
|
|
deployment_score=computed_deployment if computed_deployment is not None else je.deployment_score,
|
|
common_window_score=(
|
|
computed_common_window if computed_common_window is not None else je.common_window_score
|
|
),
|
|
common_window_summary=je.common_window_summary,
|
|
walk_forward_summary=je.walk_forward_summary,
|
|
robustness_matrix_summary=je.robustness_matrix_summary,
|
|
out_of_time_robustness_summary=je.out_of_time_robustness_summary,
|
|
train_total_return_pct=train_result.total_return_pct if train_result else None,
|
|
train_annualized_return_pct=train_result.annualized_return_pct if train_result else None,
|
|
profit_factor=test_result.profit_factor if test_result else None,
|
|
total_return_pct=test_result.total_return_pct if test_result else None,
|
|
annualized_return_pct=test_result.annualized_return_pct if test_result else None,
|
|
win_rate=test_result.win_rate if test_result else None,
|
|
sharpe_ratio=test_result.sharpe_ratio if test_result else None,
|
|
max_drawdown_pct=test_result.max_drawdown_pct if test_result else None,
|
|
trade_count=test_result.trade_count if test_result else 0,
|
|
avg_gross_exposure_pct=test_result.avg_gross_exposure_pct if test_result else None,
|
|
avg_net_exposure_pct=test_result.avg_net_exposure_pct if test_result else None,
|
|
days_in_market_pct=test_result.days_in_market_pct if test_result else None,
|
|
valid_profit_factor=valid_result.profit_factor if valid_result else None,
|
|
valid_total_return_pct=valid_result.total_return_pct if valid_result else None,
|
|
valid_annualized_return_pct=valid_result.annualized_return_pct if valid_result else None,
|
|
valid_win_rate=valid_result.win_rate if valid_result else None,
|
|
valid_sharpe_ratio=valid_result.sharpe_ratio if valid_result else None,
|
|
valid_max_drawdown_pct=valid_result.max_drawdown_pct if valid_result else None,
|
|
valid_trade_count=valid_result.trade_count if valid_result else 0,
|
|
valid_avg_gross_exposure_pct=valid_result.avg_gross_exposure_pct if valid_result else None,
|
|
valid_avg_net_exposure_pct=valid_result.avg_net_exposure_pct if valid_result else None,
|
|
valid_days_in_market_pct=valid_result.days_in_market_pct if valid_result else None,
|
|
timestamp=je.timestamp,
|
|
)
|
|
)
|
|
|
|
# Default registry / markdown ordering is public SQS v3 descending.
|
|
registry_entries.sort(
|
|
key=lambda e: (
|
|
-(e.sqs_score or 0.0),
|
|
e.stress_sqs_score is None,
|
|
-(e.stress_sqs_score or 0.0),
|
|
e.deployment_score is None,
|
|
-(e.deployment_score or 0.0),
|
|
e.rqs_score is None,
|
|
-(e.rqs_score or 0.0),
|
|
e.promotion_score is None,
|
|
-(e.promotion_score or 0.0),
|
|
)
|
|
)
|
|
|
|
registry = ExperimentRegistry(
|
|
entries=registry_entries,
|
|
updated_at=utc_now().isoformat(),
|
|
)
|
|
|
|
# Write registry JSON
|
|
registry_path.parent.mkdir(parents=True, exist_ok=True)
|
|
registry_path.write_text(registry.model_dump_json(indent=2))
|
|
|
|
# Write LEADERBOARD.md
|
|
_write_leaderboard_md(leaderboard_path, registry, entries)
|
|
|
|
logger.info("registry_rebuilt", count=len(registry_entries))
|
|
return registry
|
|
|
|
|
|
def _write_leaderboard_md(
|
|
path: Path,
|
|
registry: ExperimentRegistry,
|
|
journal_entries: list[JournalEntry],
|
|
) -> None:
|
|
visible_entries = filter_registry_entries(
|
|
registry.entries,
|
|
include_retired=False,
|
|
)
|
|
visible_recent = [
|
|
entry
|
|
for entry in reversed(journal_entries)
|
|
if (
|
|
not _is_retired_journal_entry(entry)
|
|
and _is_complete_journal_entry(entry)
|
|
and classify_strategy_family(entry.experiment_name, entry.tags) != "overlay"
|
|
)
|
|
][:5]
|
|
lines: list[str] = []
|
|
lines.append("# Strategy Improvement Leaderboard")
|
|
lines.append(f"_Updated: {registry.updated_at}_\n")
|
|
lines.append("_Default view excludes retired legacy PEAD / short-core / exact-pocket families and incomplete train-only scans. Use `fithia2 lb --include-retired` to inspect archived research._\n")
|
|
lines.append("_`SQS` below is public SQS v4: v3 deployment/WFV-first ranking plus a modest common-window capital-growth term when available. Stress OOT remains an eligibility gate. Prior v3 values remain in `experiment_registry.json` as `sqs_v3_score`._\n")
|
|
lines.append("| # | Experiment | SQS | [Tr]Ret% | [V]Ret% | [T]Ret% | [T]Ann% | [T]DD% | [T]Gross% | [T]DIM% | [T]R/G | Date |")
|
|
lines.append("|---|-----------|-----|----------|----------|----------|----------|--------|------------|----------|---------|------|")
|
|
|
|
for rank, e in enumerate(visible_entries, 1):
|
|
trret = f"{e.train_total_return_pct:+.1f}" if e.train_total_return_pct is not None else "-"
|
|
vret = f"{e.valid_total_return_pct:+.1f}" if e.valid_total_return_pct is not None else "-"
|
|
ret = f"{e.total_return_pct:+.1f}" if e.total_return_pct is not None else "-"
|
|
ann = f"{e.annualized_return_pct:+.1f}" if e.annualized_return_pct is not None else "-"
|
|
dd = f"{e.max_drawdown_pct:.1f}" if e.max_drawdown_pct is not None else "-"
|
|
gross = f"{e.avg_gross_exposure_pct:.1f}" if e.avg_gross_exposure_pct is not None else "-"
|
|
dim = f"{e.days_in_market_pct:.1f}" if e.days_in_market_pct is not None else "-"
|
|
ret_on_gross = "-"
|
|
if (
|
|
e.total_return_pct is not None
|
|
and e.avg_gross_exposure_pct is not None
|
|
and e.avg_gross_exposure_pct > 0
|
|
):
|
|
ret_on_gross = f"{e.total_return_pct / e.avg_gross_exposure_pct:.2f}"
|
|
ts = e.timestamp[:10] if e.timestamp else "-"
|
|
lines.append(
|
|
f"| {rank} | {e.experiment_name} | {e.sqs_score:.1f}"
|
|
f" | {trret} | {vret} | {ret} | {ann} | {dd} | {gross} | {dim} | {ret_on_gross} | {ts} |"
|
|
)
|
|
|
|
# Recent entries (last 5)
|
|
if visible_recent:
|
|
registry_by_id = {entry.entry_id: entry for entry in registry.entries}
|
|
lines.append("\n## Recent Entries")
|
|
for je in visible_recent:
|
|
canonical_sqs = registry_by_id.get(je.entry_id).sqs_score if je.entry_id in registry_by_id else je.sqs_score
|
|
stress_sqs = (
|
|
registry_by_id.get(je.entry_id).stress_sqs_score
|
|
if je.entry_id in registry_by_id
|
|
else je.stress_sqs_score
|
|
)
|
|
lines.append(f"### {je.entry_id} ({je.timestamp[:10]}) \u2014 {je.experiment_name}")
|
|
lines.append(f"Hypothesis: {je.hypothesis}")
|
|
stress_suffix = f", Stress SQS {stress_sqs:.1f}" if stress_sqs is not None else ""
|
|
lines.append(f"Verdict: **{je.verdict.upper()}** (SQS {canonical_sqs}{stress_suffix})")
|
|
if je.verdict_reasoning:
|
|
lines.append(f"Reasoning: {je.verdict_reasoning}")
|
|
if je.next_direction:
|
|
lines.append(f"Next: {je.next_direction}")
|
|
lines.append("")
|
|
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text("\n".join(lines) + "\n")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Run scanning helpers (for CLI record command)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def scan_runs_for_experiment(
|
|
runs_dir: Path,
|
|
experiment_name: str,
|
|
) -> dict[str, tuple[str, MetricsBundle]]:
|
|
"""Scan runs_dir for runs matching experiment_name.
|
|
|
|
Returns dict of split_name -> (run_id, MetricsBundle).
|
|
Uses manifest.json for experiment matching, metadata.json for split_name,
|
|
falls back to time-ordered grouping by config hash.
|
|
"""
|
|
from libs.backtest.domain import MetricsBundle
|
|
|
|
matches: list[tuple[Path, str, str, str, str | None]] = [] # (run_dir, run_id, config_hash, started_at, split)
|
|
|
|
for manifest_file in sorted(runs_dir.rglob("manifest.json")):
|
|
run_path = manifest_file.parent
|
|
manifest_data = json.loads(manifest_file.read_text())
|
|
if manifest_data.get("experiment_name") != experiment_name:
|
|
continue
|
|
metadata_file = run_path / "metadata.json"
|
|
metadata = json.loads(metadata_file.read_text()) if metadata_file.exists() else {}
|
|
run_id = metadata.get("run_id", run_path.name)
|
|
started_at = str(metadata.get("started_at") or metadata.get("finished_at") or "")
|
|
split_name = metadata.get("split_name")
|
|
# Extract config hash from run_id (last segment after underscore)
|
|
parts = run_path.name.split("_")
|
|
config_hash = parts[-1] if len(parts) >= 2 else ""
|
|
matches.append((run_path, run_id, config_hash, started_at, split_name))
|
|
|
|
if not matches:
|
|
return {}
|
|
|
|
results: dict[str, tuple[str, MetricsBundle]] = {}
|
|
|
|
# Check for split_name in metadata first
|
|
has_split_names = False
|
|
for _, _, _, _, split_name in matches:
|
|
if split_name is not None:
|
|
has_split_names = True
|
|
break
|
|
|
|
canonical_splits = {"train", "valid", "test"}
|
|
|
|
if has_split_names:
|
|
canonical_matches = [
|
|
(run_path, run_id, config_hash, started_at, split_name)
|
|
for run_path, run_id, config_hash, started_at, split_name in matches
|
|
if split_name in canonical_splits
|
|
]
|
|
if not canonical_matches:
|
|
return {}
|
|
|
|
latest_by_split: dict[str, tuple[str, Path, str]] = {}
|
|
for run_path, run_id, _, started_at, split_name in canonical_matches:
|
|
split = split_name or "unknown"
|
|
current = latest_by_split.get(split)
|
|
if current is None or started_at >= current[0]:
|
|
latest_by_split[split] = (started_at, run_path, run_id)
|
|
for split, (_, run_path, run_id) in latest_by_split.items():
|
|
metrics_file = run_path / "metrics" / "metrics_summary.json"
|
|
if metrics_file.exists():
|
|
metrics = MetricsBundle.model_validate_json(metrics_file.read_text())
|
|
results[split] = (run_id, metrics)
|
|
else:
|
|
# Fallback: group by config_hash, then assign train/valid/test by time order
|
|
from collections import defaultdict
|
|
groups: dict[str, list[tuple[str, Path, str]]] = defaultdict(list)
|
|
for run_path, run_id, config_hash, started_at, _ in matches:
|
|
groups[config_hash].append((started_at, run_path, run_id))
|
|
|
|
# Use the largest group (most likely the 3-split set)
|
|
biggest_group = max(groups.values(), key=len) if groups else []
|
|
split_names = ["train", "valid", "test"]
|
|
for i, (_, run_path, run_id) in enumerate(sorted(biggest_group)):
|
|
split = split_names[i] if i < len(split_names) else f"extra_{i}"
|
|
metrics_file = run_path / "metrics" / "metrics_summary.json"
|
|
if metrics_file.exists():
|
|
metrics = MetricsBundle.model_validate_json(metrics_file.read_text())
|
|
results[split] = (run_id, metrics)
|
|
|
|
return results
|
|
|
|
|
|
def check_duplicate(journal_path: Path, experiment_name: str) -> list[JournalEntry]:
|
|
"""Check if experiment_name already exists in journal."""
|
|
entries = load_journal(journal_path)
|
|
return [e for e in entries if e.experiment_name == experiment_name]
|