You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
356 lines
13 KiB
Python
356 lines
13 KiB
Python
"""Strategy improvement tracker: SQS computation, journal I/O, leaderboard."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from libs.backtest.domain import (
|
|
ConfigDelta,
|
|
ExperimentRegistry,
|
|
JournalEntry,
|
|
MetricsBundle,
|
|
RegistryEntry,
|
|
SplitResult,
|
|
SQSWeights,
|
|
)
|
|
from libs.common.logging import get_logger
|
|
from libs.common.time_utils import utc_now
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
_DEFAULT_WEIGHTS = SQSWeights()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# SQS computation (pure functions, no I/O)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _normalize(value: float | None, low: float, high: float) -> float:
|
|
"""Linear normalise *value* to 0-100 between *low* (0 pts) and *high* (100 pts)."""
|
|
if value is None:
|
|
return 0.0
|
|
if high == low:
|
|
return 50.0
|
|
score = (value - low) / (high - low) * 100.0
|
|
return max(0.0, min(100.0, score))
|
|
|
|
|
|
def _normalize_inverse(value: float | None, low: float, high: float) -> float:
|
|
"""Like _normalize but lower values are better (e.g. drawdown)."""
|
|
if value is None:
|
|
return 0.0
|
|
if high == low:
|
|
return 50.0
|
|
# low = worst (0 pts), high = best (100 pts) — but for inverse metrics
|
|
# low value = good, high value = bad. Swap the interpretation.
|
|
score = (low - value) / (low - high) * 100.0
|
|
return max(0.0, min(100.0, score))
|
|
|
|
|
|
def compute_sqs(
|
|
metrics: MetricsBundle,
|
|
weights: SQSWeights | None = None,
|
|
) -> tuple[float, dict[str, float]]:
|
|
"""Compute Strategy Quality Score from test-split metrics.
|
|
|
|
Returns (sqs_score, breakdown_dict).
|
|
"""
|
|
w = weights or _DEFAULT_WEIGHTS
|
|
|
|
# --- Profitability (40%) ---
|
|
pf_score = _normalize(metrics.profit_factor, low=0.8, high=2.0)
|
|
ret_score = _normalize(metrics.total_return_pct, low=-5.0, high=5.0)
|
|
profitability = pf_score * 0.6 + ret_score * 0.4
|
|
|
|
# --- Risk (25%) ---
|
|
dd_score = _normalize_inverse(metrics.max_drawdown_pct, low=10.0, high=1.0)
|
|
sharpe_score = _normalize(metrics.sharpe_ratio, low=-1.0, high=2.0)
|
|
risk = dd_score * 0.5 + sharpe_score * 0.5
|
|
|
|
# --- Consistency (20%) ---
|
|
wr_score = _normalize(metrics.win_rate, low=0.35, high=0.65)
|
|
mwr_score = _normalize(metrics.monthly_win_rate, low=0.30, high=0.70)
|
|
consistency = wr_score * 0.5 + mwr_score * 0.5
|
|
|
|
# --- Robustness (15%) ---
|
|
r2_score = _normalize(metrics.equity_curve_r_squared, low=0.0, high=0.80)
|
|
tc_score = _normalize(float(metrics.trade_count), low=10.0, high=100.0)
|
|
robustness = r2_score * 0.5 + tc_score * 0.5
|
|
|
|
# Weighted total
|
|
sqs = (
|
|
profitability * w.profitability
|
|
+ risk * w.risk
|
|
+ consistency * w.consistency
|
|
+ robustness * w.robustness
|
|
)
|
|
|
|
# Low-trade penalty
|
|
if metrics.trade_count < w.low_trade_penalty_threshold:
|
|
sqs *= w.low_trade_penalty_factor
|
|
|
|
sqs = round(sqs, 1)
|
|
breakdown = {
|
|
"profitability": round(profitability, 1),
|
|
"risk": round(risk, 1),
|
|
"consistency": round(consistency, 1),
|
|
"robustness": round(robustness, 1),
|
|
}
|
|
return sqs, breakdown
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helper builders
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def build_split_result(split_name: str, run_id: str, metrics: MetricsBundle) -> SplitResult:
|
|
"""Create a SplitResult from MetricsBundle."""
|
|
return SplitResult(
|
|
run_id=run_id,
|
|
trade_count=metrics.trade_count,
|
|
profit_factor=metrics.profit_factor,
|
|
total_return_pct=metrics.total_return_pct,
|
|
win_rate=metrics.win_rate,
|
|
max_drawdown_pct=metrics.max_drawdown_pct,
|
|
sharpe_ratio=metrics.sharpe_ratio,
|
|
monthly_win_rate=metrics.monthly_win_rate,
|
|
equity_curve_r_squared=metrics.equity_curve_r_squared,
|
|
)
|
|
|
|
|
|
def compute_config_delta(
|
|
current: dict[str, Any],
|
|
baseline: dict[str, Any],
|
|
baseline_name: str,
|
|
) -> ConfigDelta:
|
|
"""Compute a flat diff between two config dicts (one level deep)."""
|
|
changes: dict[str, str] = {}
|
|
_diff_recursive(baseline, current, prefix="", changes=changes)
|
|
return ConfigDelta(base_experiment=baseline_name, changes=changes)
|
|
|
|
|
|
def _diff_recursive(
|
|
old: dict[str, Any],
|
|
new: dict[str, Any],
|
|
prefix: str,
|
|
changes: dict[str, str],
|
|
) -> None:
|
|
all_keys = set(old.keys()) | set(new.keys())
|
|
for key in sorted(all_keys):
|
|
full_key = f"{prefix}{key}" if not prefix else f"{prefix}.{key}"
|
|
old_val = old.get(key)
|
|
new_val = new.get(key)
|
|
if isinstance(old_val, dict) and isinstance(new_val, dict):
|
|
_diff_recursive(old_val, new_val, full_key, changes)
|
|
elif old_val != new_val:
|
|
changes[full_key] = f"{old_val} \u2192 {new_val}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Journal I/O
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def get_next_entry_id(journal_path: Path) -> str:
|
|
"""Return next entry ID like 'IMP-0001'."""
|
|
entries = load_journal(journal_path)
|
|
return f"IMP-{len(entries) + 1:04d}"
|
|
|
|
|
|
def append_journal_entry(journal_path: Path, entry: JournalEntry) -> None:
|
|
"""Append a single JournalEntry as one JSON line."""
|
|
journal_path.parent.mkdir(parents=True, exist_ok=True)
|
|
with open(journal_path, "a") as f:
|
|
f.write(entry.model_dump_json() + "\n")
|
|
logger.info("journal_entry_appended", entry_id=entry.entry_id, experiment=entry.experiment_name)
|
|
|
|
|
|
def load_journal(journal_path: Path) -> list[JournalEntry]:
|
|
"""Load all journal entries from JSONL file."""
|
|
if not journal_path.exists():
|
|
return []
|
|
entries: list[JournalEntry] = []
|
|
for line in journal_path.read_text().strip().splitlines():
|
|
line = line.strip()
|
|
if line:
|
|
entries.append(JournalEntry.model_validate_json(line))
|
|
return entries
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Registry / Leaderboard
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def rebuild_registry(
|
|
journal_path: Path,
|
|
registry_path: Path,
|
|
leaderboard_path: Path,
|
|
) -> ExperimentRegistry:
|
|
"""Rebuild experiment_registry.json and LEADERBOARD.md from journal."""
|
|
entries = load_journal(journal_path)
|
|
|
|
registry_entries: list[RegistryEntry] = []
|
|
for je in entries:
|
|
test_result = je.results.get("test")
|
|
registry_entries.append(
|
|
RegistryEntry(
|
|
entry_id=je.entry_id,
|
|
experiment_name=je.experiment_name,
|
|
sqs_score=je.sqs_score or 0.0,
|
|
profit_factor=test_result.profit_factor if test_result else None,
|
|
total_return_pct=test_result.total_return_pct if test_result else None,
|
|
win_rate=test_result.win_rate if test_result else None,
|
|
sharpe_ratio=test_result.sharpe_ratio if test_result else None,
|
|
max_drawdown_pct=test_result.max_drawdown_pct if test_result else None,
|
|
trade_count=test_result.trade_count if test_result else 0,
|
|
timestamp=je.timestamp,
|
|
)
|
|
)
|
|
|
|
# Sort by SQS descending
|
|
registry_entries.sort(key=lambda e: e.sqs_score, reverse=True)
|
|
|
|
registry = ExperimentRegistry(
|
|
entries=registry_entries,
|
|
updated_at=utc_now().isoformat(),
|
|
)
|
|
|
|
# Write registry JSON
|
|
registry_path.parent.mkdir(parents=True, exist_ok=True)
|
|
registry_path.write_text(registry.model_dump_json(indent=2))
|
|
|
|
# Write LEADERBOARD.md
|
|
_write_leaderboard_md(leaderboard_path, registry, entries)
|
|
|
|
logger.info("registry_rebuilt", count=len(registry_entries))
|
|
return registry
|
|
|
|
|
|
def _write_leaderboard_md(
|
|
path: Path,
|
|
registry: ExperimentRegistry,
|
|
journal_entries: list[JournalEntry],
|
|
) -> None:
|
|
lines: list[str] = []
|
|
lines.append("# Strategy Improvement Leaderboard")
|
|
lines.append(f"_Updated: {registry.updated_at}_\n")
|
|
lines.append("| # | Experiment | SQS | PF | Ret% | WR | Sharpe | DD% | Trades | Date |")
|
|
lines.append("|---|-----------|-----|-----|------|-----|--------|-----|--------|------|")
|
|
|
|
for rank, e in enumerate(registry.entries, 1):
|
|
pf = f"{e.profit_factor:.2f}" if e.profit_factor is not None else "-"
|
|
ret = f"{e.total_return_pct:+.1f}" if e.total_return_pct is not None else "-"
|
|
wr = f"{e.win_rate:.0%}" if e.win_rate is not None else "-"
|
|
sharpe = f"{e.sharpe_ratio:.1f}" if e.sharpe_ratio is not None else "-"
|
|
dd = f"{e.max_drawdown_pct:.1f}" if e.max_drawdown_pct is not None else "-"
|
|
ts = e.timestamp[:10] if e.timestamp else "-"
|
|
lines.append(
|
|
f"| {rank} | {e.experiment_name} | {e.sqs_score:.1f} | {pf} | {ret} | {wr} | {sharpe} | {dd} | {e.trade_count} | {ts} |"
|
|
)
|
|
|
|
# Recent entries (last 5)
|
|
recent = list(reversed(journal_entries))[:5]
|
|
if recent:
|
|
lines.append("\n## Recent Entries")
|
|
for je in recent:
|
|
lines.append(f"### {je.entry_id} ({je.timestamp[:10]}) \u2014 {je.experiment_name}")
|
|
lines.append(f"Hypothesis: {je.hypothesis}")
|
|
lines.append(f"Verdict: **{je.verdict.upper()}** (SQS {je.sqs_score})")
|
|
if je.verdict_reasoning:
|
|
lines.append(f"Reasoning: {je.verdict_reasoning}")
|
|
if je.next_direction:
|
|
lines.append(f"Next: {je.next_direction}")
|
|
lines.append("")
|
|
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text("\n".join(lines) + "\n")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Run scanning helpers (for CLI record command)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def scan_runs_for_experiment(
|
|
runs_dir: Path,
|
|
experiment_name: str,
|
|
) -> dict[str, tuple[str, MetricsBundle]]:
|
|
"""Scan runs_dir for runs matching experiment_name.
|
|
|
|
Returns dict of split_name -> (run_id, MetricsBundle).
|
|
Uses manifest.json for experiment matching, metadata.json for split_name,
|
|
falls back to time-ordered grouping by config hash.
|
|
"""
|
|
from libs.backtest.domain import MetricsBundle
|
|
|
|
matches: list[tuple[Path, str, str]] = [] # (run_dir, run_id, config_hash)
|
|
|
|
for run_path in sorted(runs_dir.iterdir()):
|
|
if not run_path.is_dir():
|
|
continue
|
|
manifest_file = run_path / "manifest.json"
|
|
if not manifest_file.exists():
|
|
continue
|
|
manifest_data = json.loads(manifest_file.read_text())
|
|
if manifest_data.get("experiment_name") != experiment_name:
|
|
continue
|
|
metadata_file = run_path / "metadata.json"
|
|
metadata = json.loads(metadata_file.read_text()) if metadata_file.exists() else {}
|
|
run_id = metadata.get("run_id", run_path.name)
|
|
# Extract config hash from run_id (last segment after underscore)
|
|
parts = run_path.name.split("_")
|
|
config_hash = parts[-1] if len(parts) >= 2 else ""
|
|
matches.append((run_path, run_id, config_hash))
|
|
|
|
if not matches:
|
|
return {}
|
|
|
|
results: dict[str, tuple[str, MetricsBundle]] = {}
|
|
|
|
# Check for split_name in metadata first
|
|
has_split_names = False
|
|
for run_path, run_id, _ in matches:
|
|
metadata_file = run_path / "metadata.json"
|
|
if metadata_file.exists():
|
|
metadata = json.loads(metadata_file.read_text())
|
|
if "split_name" in metadata:
|
|
has_split_names = True
|
|
break
|
|
|
|
if has_split_names:
|
|
for run_path, run_id, _ in matches:
|
|
metadata = json.loads((run_path / "metadata.json").read_text())
|
|
split = metadata.get("split_name", "unknown")
|
|
metrics_file = run_path / "metrics" / "metrics_summary.json"
|
|
if metrics_file.exists():
|
|
metrics = MetricsBundle.model_validate_json(metrics_file.read_text())
|
|
results[split] = (run_id, metrics)
|
|
else:
|
|
# Fallback: group by config_hash, then assign train/valid/test by time order
|
|
from collections import defaultdict
|
|
groups: dict[str, list[tuple[Path, str]]] = defaultdict(list)
|
|
for run_path, run_id, config_hash in matches:
|
|
groups[config_hash].append((run_path, run_id))
|
|
|
|
# Use the largest group (most likely the 3-split set)
|
|
biggest_group = max(groups.values(), key=len) if groups else []
|
|
split_names = ["train", "valid", "test"]
|
|
for i, (run_path, run_id) in enumerate(biggest_group):
|
|
split = split_names[i] if i < len(split_names) else f"extra_{i}"
|
|
metrics_file = run_path / "metrics" / "metrics_summary.json"
|
|
if metrics_file.exists():
|
|
metrics = MetricsBundle.model_validate_json(metrics_file.read_text())
|
|
results[split] = (run_id, metrics)
|
|
|
|
return results
|
|
|
|
|
|
def check_duplicate(journal_path: Path, experiment_name: str) -> list[JournalEntry]:
|
|
"""Check if experiment_name already exists in journal."""
|
|
entries = load_journal(journal_path)
|
|
return [e for e in entries if e.experiment_name == experiment_name]
|