You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
446 lines
13 KiB
Python
446 lines
13 KiB
Python
"""Overfitting detection tests for quantitative trading strategies.
|
|
|
|
Implements five academic/practitioner tests:
|
|
1. DSR — Deflated Sharpe Ratio (Bailey & Lopez de Prado, 2014)
|
|
2. PBO — Probability of Backtest Overfitting via CSCV (Bailey et al., 2017)
|
|
3. SENS — Parameter Sensitivity / Plateau Analysis
|
|
4. MC — Monte Carlo Noise Injection
|
|
5. PERM — Walk-Forward Permutation Test
|
|
|
|
Usage:
|
|
from libs.backtest.overfit import run_overfit_check
|
|
report = run_overfit_check("return_max_long_v7.70", initial_equity=10_000)
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import json
|
|
import math
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from itertools import combinations
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
|
|
import numpy as np
|
|
import pyarrow.parquet as pq
|
|
from scipy.stats import gaussian_kde, kurtosis, norm, skew
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Result models
|
|
# ---------------------------------------------------------------------------
|
|
@dataclass
|
|
class DSRResult:
|
|
observed_sharpe: float
|
|
expected_null_sharpe: float
|
|
deflated_sharpe: float
|
|
dsr_pvalue: float
|
|
n_trials: int
|
|
sharpe_haircut_pct: float
|
|
min_track_record_days: float
|
|
data_days: int
|
|
all_sharpes: list[float] = field(default_factory=list)
|
|
verdict: str = ""
|
|
|
|
def __post_init__(self):
|
|
if not self.verdict:
|
|
if self.dsr_pvalue > 0.95:
|
|
self.verdict = "PASS"
|
|
elif self.dsr_pvalue > 0.50:
|
|
self.verdict = "WARN"
|
|
else:
|
|
self.verdict = "FAIL"
|
|
|
|
|
|
@dataclass
|
|
class PBOResult:
|
|
pbo_probability: float
|
|
n_strategies: int
|
|
n_subsets: int
|
|
n_combinations: int
|
|
median_oos_rank: float
|
|
oos_above_50_pct: float
|
|
oos_sharpe_positive_pct: float
|
|
logit_mean: float
|
|
verdict: str = ""
|
|
|
|
def __post_init__(self):
|
|
if not self.verdict:
|
|
if self.pbo_probability < 0.10:
|
|
self.verdict = "PASS"
|
|
elif self.pbo_probability < 0.30:
|
|
self.verdict = "WARN"
|
|
else:
|
|
self.verdict = "FAIL"
|
|
|
|
|
|
@dataclass
|
|
class ParamSensitivityResult:
|
|
param_name: str
|
|
values_tested: list[float]
|
|
sharpe_values: list[float]
|
|
plateau_score: float
|
|
verdict: str = ""
|
|
|
|
def __post_init__(self):
|
|
if not self.verdict:
|
|
if self.plateau_score > 0.70:
|
|
self.verdict = "PASS"
|
|
elif self.plateau_score > 0.40:
|
|
self.verdict = "WARN"
|
|
else:
|
|
self.verdict = "FAIL"
|
|
|
|
|
|
@dataclass
|
|
class MonteCarloResult:
|
|
baseline_sharpe: float
|
|
noise_level_pct: float
|
|
n_iterations: int
|
|
median_noised_sharpe: float
|
|
p05_sharpe: float
|
|
degradation_pct: float
|
|
all_sharpes: list[float] = field(default_factory=list)
|
|
verdict: str = ""
|
|
|
|
def __post_init__(self):
|
|
if not self.verdict:
|
|
if self.degradation_pct < 15:
|
|
self.verdict = "PASS"
|
|
elif self.degradation_pct < 35:
|
|
self.verdict = "WARN"
|
|
else:
|
|
self.verdict = "FAIL"
|
|
|
|
|
|
@dataclass
|
|
class PermutationResult:
|
|
observed_sharpe: float
|
|
n_permutations: int
|
|
p_value: float
|
|
null_median_sharpe: float
|
|
null_p95_sharpe: float
|
|
verdict: str = ""
|
|
|
|
def __post_init__(self):
|
|
if not self.verdict:
|
|
if self.p_value < 0.01:
|
|
self.verdict = "PASS"
|
|
elif self.p_value < 0.05:
|
|
self.verdict = "WARN"
|
|
else:
|
|
self.verdict = "FAIL"
|
|
|
|
|
|
@dataclass
|
|
class OverfitReport:
|
|
experiment_name: str
|
|
experiment_id: int | None
|
|
overall_verdict: str
|
|
overall_score: float
|
|
dsr: DSRResult | None = None
|
|
pbo: PBOResult | None = None
|
|
param_sensitivity: list[ParamSensitivityResult] = field(default_factory=list)
|
|
monte_carlo: MonteCarloResult | None = None
|
|
permutation: PermutationResult | None = None
|
|
timestamp: str = ""
|
|
elapsed_seconds: float = 0
|
|
|
|
def __post_init__(self):
|
|
if not self.timestamp:
|
|
self.timestamp = datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test 1: Deflated Sharpe Ratio
|
|
# ---------------------------------------------------------------------------
|
|
def compute_dsr(
|
|
daily_returns: np.ndarray,
|
|
observed_sharpe: float,
|
|
n_trials: int,
|
|
all_sharpes: list[float] | None = None,
|
|
) -> DSRResult:
|
|
T = len(daily_returns)
|
|
g3 = float(skew(daily_returns))
|
|
g4 = float(kurtosis(daily_returns, fisher=False))
|
|
gamma = 0.5772156649
|
|
|
|
if all_sharpes and len(all_sharpes) > 1:
|
|
var_sr = float(np.var(all_sharpes, ddof=1))
|
|
else:
|
|
var_sr = 1.0 / T
|
|
|
|
sr0 = math.sqrt(var_sr) * (
|
|
(1 - gamma) * norm.ppf(1 - 1.0 / max(n_trials, 2))
|
|
+ gamma * norm.ppf(1 - 1.0 / (max(n_trials, 2) * math.e))
|
|
)
|
|
|
|
numer = (observed_sharpe - sr0) * math.sqrt(T - 1)
|
|
denom = math.sqrt(max(1e-10, 1 - g3 * observed_sharpe + (g4 - 1) / 4 * observed_sharpe**2))
|
|
dsr = float(norm.cdf(numer / denom))
|
|
|
|
haircut = sr0 / observed_sharpe * 100 if observed_sharpe > 0 else 100.0
|
|
surviving_sr = max(0, observed_sharpe - sr0)
|
|
|
|
min_trl = float("inf")
|
|
if surviving_sr > 0:
|
|
min_trl = 1 + (1 - g3 * surviving_sr + (g4 - 1) / 4 * surviving_sr**2) * (
|
|
norm.ppf(0.95) / surviving_sr
|
|
) ** 2
|
|
|
|
return DSRResult(
|
|
observed_sharpe=observed_sharpe,
|
|
expected_null_sharpe=sr0,
|
|
deflated_sharpe=surviving_sr,
|
|
dsr_pvalue=dsr,
|
|
n_trials=n_trials,
|
|
sharpe_haircut_pct=haircut,
|
|
min_track_record_days=min_trl,
|
|
data_days=T,
|
|
all_sharpes=all_sharpes or [],
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test 2: PBO via CSCV
|
|
# ---------------------------------------------------------------------------
|
|
def compute_pbo(
|
|
returns_matrix: np.ndarray,
|
|
n_subsets: int = 10,
|
|
) -> PBOResult:
|
|
T, N = returns_matrix.shape
|
|
chunk = T // n_subsets
|
|
blocks = [returns_matrix[i * chunk : (i + 1) * chunk] for i in range(n_subsets)]
|
|
if T > n_subsets * chunk:
|
|
blocks[-1] = returns_matrix[(n_subsets - 1) * chunk :]
|
|
|
|
combos = list(combinations(range(n_subsets), n_subsets // 2))
|
|
|
|
lambdas = []
|
|
oos_ranks = []
|
|
oos_sharpes = []
|
|
|
|
for combo in combos:
|
|
test_idx = sorted(set(range(n_subsets)) - set(combo))
|
|
train = np.concatenate([blocks[i] for i in combo])
|
|
test = np.concatenate([blocks[i] for i in test_idx])
|
|
|
|
train_std = train.std(axis=0, ddof=1)
|
|
train_std[train_std == 0] = 1e-10
|
|
train_sr = train.mean(axis=0) / train_std
|
|
|
|
test_std = test.std(axis=0, ddof=1)
|
|
test_std[test_std == 0] = 1e-10
|
|
test_sr = test.mean(axis=0) / test_std
|
|
|
|
best_is = int(np.argmax(train_sr))
|
|
rank = float((test_sr < test_sr[best_is]).sum()) / N
|
|
rank = max(0.01, min(0.99, rank))
|
|
oos_ranks.append(rank)
|
|
lambdas.append(math.log(rank / (1 - rank)))
|
|
oos_sharpes.append(float(test_sr[best_is]))
|
|
|
|
lambdas_arr = np.array(lambdas)
|
|
oos_ranks_arr = np.array(oos_ranks)
|
|
|
|
kde = gaussian_kde(lambdas_arr)
|
|
pbo = float(kde.integrate_box_1d(-np.inf, 0))
|
|
|
|
return PBOResult(
|
|
pbo_probability=pbo,
|
|
n_strategies=N,
|
|
n_subsets=n_subsets,
|
|
n_combinations=len(combos),
|
|
median_oos_rank=float(np.median(oos_ranks_arr)),
|
|
oos_above_50_pct=float((oos_ranks_arr > 0.5).mean() * 100),
|
|
oos_sharpe_positive_pct=float((np.array(oos_sharpes) > 0).mean() * 100),
|
|
logit_mean=float(lambdas_arr.mean()),
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test 3: Parameter Sensitivity
|
|
# ---------------------------------------------------------------------------
|
|
def compute_param_sensitivity(
|
|
run_backtest_fn: Callable[[dict[str, Any]], float],
|
|
base_config: dict[str, Any],
|
|
param_path: str,
|
|
values: list[float],
|
|
) -> ParamSensitivityResult:
|
|
sharpes = []
|
|
for val in values:
|
|
cfg = copy.deepcopy(base_config)
|
|
_set_nested(cfg, param_path, val)
|
|
sr = run_backtest_fn(cfg)
|
|
sharpes.append(sr)
|
|
|
|
sharpes_arr = np.array(sharpes)
|
|
if sharpes_arr.mean() > 0:
|
|
cv = float(sharpes_arr.std(ddof=1) / sharpes_arr.mean())
|
|
plateau = max(0, 1 - cv)
|
|
else:
|
|
plateau = 0.0
|
|
|
|
return ParamSensitivityResult(
|
|
param_name=param_path,
|
|
values_tested=values,
|
|
sharpe_values=sharpes,
|
|
plateau_score=plateau,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test 4: Monte Carlo Noise Injection
|
|
# ---------------------------------------------------------------------------
|
|
def compute_monte_carlo(
|
|
run_backtest_fn: Callable[[float | None], float],
|
|
baseline_sharpe: float,
|
|
noise_pct: float = 0.5,
|
|
n_iterations: int = 50,
|
|
) -> MonteCarloResult:
|
|
noised_sharpes = []
|
|
for i in range(n_iterations):
|
|
sr = run_backtest_fn(noise_pct / 100.0)
|
|
noised_sharpes.append(sr)
|
|
|
|
arr = np.array(noised_sharpes)
|
|
median_sr = float(np.median(arr))
|
|
p05 = float(np.percentile(arr, 5))
|
|
degradation = (1 - median_sr / baseline_sharpe) * 100 if baseline_sharpe > 0 else 100
|
|
|
|
return MonteCarloResult(
|
|
baseline_sharpe=baseline_sharpe,
|
|
noise_level_pct=noise_pct,
|
|
n_iterations=n_iterations,
|
|
median_noised_sharpe=median_sr,
|
|
p05_sharpe=p05,
|
|
degradation_pct=max(0, degradation),
|
|
all_sharpes=noised_sharpes,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test 5: Walk-Forward Permutation
|
|
# ---------------------------------------------------------------------------
|
|
def compute_permutation(
|
|
run_wfv_fn: Callable[[bool], float],
|
|
observed_sharpe: float,
|
|
n_permutations: int = 200,
|
|
) -> PermutationResult:
|
|
null_sharpes = []
|
|
for i in range(n_permutations):
|
|
sr = run_wfv_fn(True)
|
|
null_sharpes.append(sr)
|
|
|
|
arr = np.array(null_sharpes)
|
|
p_value = float((arr >= observed_sharpe).mean())
|
|
|
|
return PermutationResult(
|
|
observed_sharpe=observed_sharpe,
|
|
n_permutations=n_permutations,
|
|
p_value=p_value,
|
|
null_median_sharpe=float(np.median(arr)),
|
|
null_p95_sharpe=float(np.percentile(arr, 95)),
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Overall scoring
|
|
# ---------------------------------------------------------------------------
|
|
WEIGHTS = {
|
|
"dsr": 0.25,
|
|
"pbo": 0.25,
|
|
"sensitivity": 0.15,
|
|
"mc": 0.20,
|
|
"permutation": 0.15,
|
|
}
|
|
|
|
|
|
def _test_score(result: Any) -> float:
|
|
if result is None:
|
|
return -1
|
|
if isinstance(result, list):
|
|
if not result:
|
|
return -1
|
|
return float(np.mean([_test_score(r) for r in result]))
|
|
v = result.verdict
|
|
if v == "PASS":
|
|
return 100
|
|
elif v == "WARN":
|
|
return 55
|
|
else:
|
|
return 15
|
|
|
|
|
|
def compute_overall_score(report: OverfitReport) -> tuple[float, str]:
|
|
scores = {
|
|
"dsr": _test_score(report.dsr),
|
|
"pbo": _test_score(report.pbo),
|
|
"sensitivity": _test_score(report.param_sensitivity),
|
|
"mc": _test_score(report.monte_carlo),
|
|
"permutation": _test_score(report.permutation),
|
|
}
|
|
|
|
total_weight = sum(w for k, w in WEIGHTS.items() if scores[k] >= 0)
|
|
if total_weight == 0:
|
|
return 0, "UNKNOWN"
|
|
|
|
weighted = sum(scores[k] * WEIGHTS[k] for k in WEIGHTS if scores[k] >= 0)
|
|
overall = weighted / total_weight
|
|
|
|
if overall >= 70:
|
|
verdict = "PASS"
|
|
elif overall >= 40:
|
|
verdict = "WARN"
|
|
else:
|
|
verdict = "FAIL"
|
|
|
|
return overall, verdict
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
def _set_nested(d: dict, path: str, value: Any) -> None:
|
|
keys = path.split(".")
|
|
for k in keys[:-1]:
|
|
d = d.setdefault(k, {})
|
|
d[keys[-1]] = value
|
|
|
|
|
|
def load_equity_curve_returns(run_dir: Path) -> np.ndarray | None:
|
|
eq_path = run_dir / "artifacts" / "daily_equity_curve.parquet"
|
|
if not eq_path.exists():
|
|
return None
|
|
eq = pq.read_table(str(eq_path))
|
|
equities = eq.column("equity").to_pylist()
|
|
if len(equities) < 2:
|
|
return None
|
|
returns = np.diff(equities) / np.array(equities[:-1])
|
|
return returns
|
|
|
|
|
|
def load_sibling_returns_matrix(
|
|
experiment_name: str,
|
|
runs_root: Path = Path("runs"),
|
|
) -> tuple[np.ndarray | None, list[str]]:
|
|
"""Load daily return series from all sibling CW runs into a (T, N) matrix."""
|
|
strats: dict[str, np.ndarray] = {}
|
|
|
|
for d in sorted(runs_root.glob("v7.*_10k_cw")):
|
|
name = d.name.replace("_10k_cw", "")
|
|
for r in d.glob("bt_*"):
|
|
rets = load_equity_curve_returns(r)
|
|
if rets is not None and len(rets) > 100:
|
|
strats[name] = rets
|
|
|
|
if len(strats) < 3:
|
|
return None, []
|
|
|
|
min_len = min(len(r) for r in strats.values())
|
|
names = sorted(strats.keys())
|
|
matrix = np.column_stack([strats[n][:min_len] for n in names])
|
|
return matrix, names
|