yirugi
/
fithia2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
main
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from '090bfa8e36'
${ noResults }
fithia2/libs/backtest/tracker.py

761 lines
30 KiB
Python
Raw Blame History

"""Strategy improvement tracker: SQS computation, journal I/O, leaderboard."""
from __future__ import annotations
import contextlib
import functools
import fcntl
import json
from pathlib import Path
from typing import Any, Iterator
from libs.backtest.domain import (
ConfigDelta,
ExperimentRegistry,
JournalEntry,
MetricsBundle,
PromotionScoreWeights,
RegistryEntry,
SplitResult,
SQSWeights,
SQSv2Weights,
UnifiedScoreWeights,
)
from libs.common.logging import get_logger
from libs.common.time_utils import utc_now
logger = get_logger(__name__)
_DEFAULT_WEIGHTS = SQSWeights()
_DEFAULT_V2_WEIGHTS = SQSv2Weights()
_DEFAULT_PROMOTION_WEIGHTS = PromotionScoreWeights()
_DEFAULT_UNIFIED_WEIGHTS = UnifiedScoreWeights()
# ---------------------------------------------------------------------------
# SQS computation (pure functions, no I/O)
# ---------------------------------------------------------------------------
def _normalize(value: float | None, low: float, high: float) -> float:
"""Linear normalise *value* to 0-100 between *low* (0 pts) and *high* (100 pts)."""
if value is None:
return 0.0
if high == low:
return 50.0
score = (value - low) / (high - low) * 100.0
return max(0.0, min(100.0, score))
def _normalize_inverse(value: float | None, low: float, high: float) -> float:
"""Like _normalize but lower values are better (e.g. drawdown)."""
if value is None:
return 0.0
if high == low:
return 50.0
# low = worst (0 pts), high = best (100 pts) — but for inverse metrics
# low value = good, high value = bad. Swap the interpretation.
score = (low - value) / (low - high) * 100.0
return max(0.0, min(100.0, score))
def _normalize_band(
value: float | None,
low_bad: float,
low_good: float,
high_good: float,
high_bad: float,
) -> float:
"""Score 0-100 with an optimal middle band."""
if value is None:
return 0.0
if value <= low_bad or value >= high_bad:
return 0.0
if low_good <= value <= high_good:
return 100.0
if value < low_good:
return (value - low_bad) / (low_good - low_bad) * 100.0
return (high_bad - value) / (high_bad - high_good) * 100.0
def _calibrate_sqs(value: float | None) -> float | None:
"""Compress raw integrated scores into a harsher absolute-looking range."""
if value is None:
return None
return round(max(0.0, value * 0.70 - 7.5), 1)
def _apply_single_split_penalty(value: float | None) -> float | None:
"""Discount scores that have no valid/test confirmation pair."""
if value is None:
return None
return round(value * 0.80, 1)
def compute_sqs(
metrics: MetricsBundle,
weights: SQSWeights | None = None,
) -> tuple[float, dict[str, float]]:
"""Compute Strategy Quality Score from test-split metrics.
Returns (sqs_score, breakdown_dict).
"""
w = weights or _DEFAULT_WEIGHTS
# --- Profitability (40%) ---
pf_score = _normalize(metrics.profit_factor, low=0.8, high=2.0)
ret_score = _normalize(metrics.total_return_pct, low=-5.0, high=5.0)
profitability = pf_score * 0.6 + ret_score * 0.4
# --- Risk (25%) ---
dd_score = _normalize_inverse(metrics.max_drawdown_pct, low=10.0, high=1.0)
sharpe_score = _normalize(metrics.sharpe_ratio, low=-1.0, high=2.0)
risk = dd_score * 0.5 + sharpe_score * 0.5
# --- Consistency (20%) ---
wr_score = _normalize(metrics.win_rate, low=0.35, high=0.65)
mwr_score = _normalize(metrics.monthly_win_rate, low=0.30, high=0.70)
consistency = wr_score * 0.5 + mwr_score * 0.5
# --- Robustness (15%) ---
r2_score = _normalize(metrics.equity_curve_r_squared, low=0.0, high=0.80)
tc_score = _normalize(float(metrics.trade_count), low=10.0, high=100.0)
robustness = r2_score * 0.5 + tc_score * 0.5
# Weighted total
sqs = (
profitability * w.profitability
+ risk * w.risk
+ consistency * w.consistency
+ robustness * w.robustness
)
# Low-trade penalty
if metrics.trade_count < w.low_trade_penalty_threshold:
sqs *= w.low_trade_penalty_factor
sqs = round(sqs, 1)
breakdown = {
"profitability": round(profitability, 1),
"risk": round(risk, 1),
"consistency": round(consistency, 1),
"robustness": round(robustness, 1),
}
return sqs, breakdown
def compute_sqs_v2(
metrics: MetricsBundle,
weights: SQSv2Weights | None = None,
) -> tuple[float | None, dict[str, float]]:
"""Compute SQS v2, adding a capital-efficiency sleeve to the score."""
if metrics.avg_gross_exposure_pct is None or metrics.days_in_market_pct is None:
return None, {}
w = weights or _DEFAULT_V2_WEIGHTS
pf_score = _normalize(metrics.profit_factor, low=0.8, high=2.0)
ret_score = _normalize(metrics.total_return_pct, low=-5.0, high=5.0)
profitability = pf_score * 0.6 + ret_score * 0.4
dd_score = _normalize_inverse(metrics.max_drawdown_pct, low=10.0, high=1.0)
sharpe_score = _normalize(metrics.sharpe_ratio, low=-1.0, high=2.0)
risk = dd_score * 0.5 + sharpe_score * 0.5
wr_score = _normalize(metrics.win_rate, low=0.35, high=0.65)
mwr_score = _normalize(metrics.monthly_win_rate, low=0.30, high=0.70)
consistency = wr_score * 0.5 + mwr_score * 0.5
r2_score = _normalize(metrics.equity_curve_r_squared, low=0.0, high=0.80)
tc_score = _normalize(float(metrics.trade_count), low=10.0, high=100.0)
robustness = r2_score * 0.5 + tc_score * 0.5
return_on_gross_exposure = None
if metrics.total_return_pct is not None and metrics.avg_gross_exposure_pct > 0:
return_on_gross_exposure = metrics.total_return_pct / metrics.avg_gross_exposure_pct
roe_score = _normalize(return_on_gross_exposure, low=0.0, high=0.50)
dim_score = _normalize_band(
metrics.days_in_market_pct,
low_bad=10.0,
low_good=40.0,
high_good=80.0,
high_bad=100.0,
)
capital_efficiency = roe_score * 0.7 + dim_score * 0.3
sqs_v2 = (
profitability * w.profitability
+ risk * w.risk
+ consistency * w.consistency
+ robustness * w.robustness
+ capital_efficiency * w.capital_efficiency
)
if metrics.trade_count < w.low_trade_penalty_threshold:
sqs_v2 *= w.low_trade_penalty_factor
sqs_v2 = round(sqs_v2, 1)
breakdown = {
"profitability": round(profitability, 1),
"risk": round(risk, 1),
"consistency": round(consistency, 1),
"robustness": round(robustness, 1),
"capital_efficiency": round(capital_efficiency, 1),
}
return sqs_v2, breakdown
# ---------------------------------------------------------------------------
# Helper builders
# ---------------------------------------------------------------------------
def build_split_result(split_name: str, run_id: str, metrics: MetricsBundle) -> SplitResult:
"""Create a SplitResult from MetricsBundle."""
return SplitResult(
run_id=run_id,
trade_count=metrics.trade_count,
profit_factor=metrics.profit_factor,
total_return_pct=metrics.total_return_pct,
win_rate=metrics.win_rate,
max_drawdown_pct=metrics.max_drawdown_pct,
sharpe_ratio=metrics.sharpe_ratio,
monthly_win_rate=metrics.monthly_win_rate,
equity_curve_r_squared=metrics.equity_curve_r_squared,
avg_gross_exposure_pct=metrics.avg_gross_exposure_pct,
avg_net_exposure_pct=metrics.avg_net_exposure_pct,
days_in_market_pct=metrics.days_in_market_pct,
)
def _metrics_from_split_result(result: SplitResult | None) -> MetricsBundle | None:
if result is None:
return None
payload = result.model_dump(exclude={"run_id"})
return MetricsBundle.model_validate(payload)
@functools.lru_cache(maxsize=512)
def _load_run_metrics_summary(run_id: str) -> dict[str, Any] | None:
metrics_path = Path("runs") / run_id / "metrics" / "metrics_summary.json"
if not metrics_path.exists():
return None
summary = json.loads(metrics_path.read_text())
needed_fields = (
"avg_gross_exposure_pct",
"avg_net_exposure_pct",
"days_in_market_pct",
)
if all(summary.get(field) is not None for field in needed_fields):
return summary
equity_curve_path = Path("runs") / run_id / "artifacts" / "daily_equity_curve.parquet"
if not equity_curve_path.exists():
return summary
try:
import pyarrow.parquet as pq
rows = pq.read_table(
equity_curve_path,
columns=["equity", "gross_exposure", "net_exposure"],
).to_pylist()
except Exception:
return summary
exposure_rows = [row for row in rows if float(row["equity"]) > 0]
if not exposure_rows:
return summary
if summary.get("avg_gross_exposure_pct") is None:
summary["avg_gross_exposure_pct"] = sum(
float(row["gross_exposure"]) / float(row["equity"]) * 100.0
for row in exposure_rows
) / len(exposure_rows)
if summary.get("avg_net_exposure_pct") is None:
summary["avg_net_exposure_pct"] = sum(
float(row["net_exposure"]) / float(row["equity"]) * 100.0
for row in exposure_rows
) / len(exposure_rows)
if summary.get("days_in_market_pct") is None:
summary["days_in_market_pct"] = (
sum(1 for row in rows if float(row["gross_exposure"]) > 0) / len(rows) * 100.0
)
return summary
def _hydrate_split_result(result: SplitResult | None) -> SplitResult | None:
if result is None:
return None
needed_fields = (
"avg_gross_exposure_pct",
"avg_net_exposure_pct",
"days_in_market_pct",
)
if all(getattr(result, field) is not None for field in needed_fields):
return result
summary = _load_run_metrics_summary(result.run_id)
if summary is None:
return result
updates = {
field: summary.get(field)
for field in needed_fields
if getattr(result, field) is None and summary.get(field) is not None
}
if not updates:
return result
return result.model_copy(update=updates)
def _compute_split_quality_score(
result: SplitResult | None,
) -> tuple[float | None, dict[str, float], str | None]:
metrics = _metrics_from_split_result(_hydrate_split_result(result))
if metrics is None:
return None, {}, None
sqs_v2_score, sqs_v2_breakdown = compute_sqs_v2(metrics)
if sqs_v2_score is not None:
return sqs_v2_score, sqs_v2_breakdown, "sqs_v2"
sqs_score, sqs_breakdown = compute_sqs(metrics)
return sqs_score, sqs_breakdown, "sqs"
def compute_unified_split_quality(
metrics: MetricsBundle,
weights: UnifiedScoreWeights | None = None,
) -> tuple[float | None, dict[str, float]]:
"""Compute a stricter split-level quality score."""
if metrics.avg_gross_exposure_pct is None or metrics.days_in_market_pct is None:
return None, {}
w = weights or _DEFAULT_UNIFIED_WEIGHTS
pf_score = _normalize(metrics.profit_factor, low=1.0, high=3.0)
ret_score = _normalize(metrics.total_return_pct, low=0.0, high=3.0)
profitability = pf_score * 0.5 + ret_score * 0.5
dd_score = _normalize_inverse(metrics.max_drawdown_pct, low=5.0, high=0.5)
sharpe_score = _normalize(metrics.sharpe_ratio, low=0.0, high=3.5)
risk = dd_score * 0.5 + sharpe_score * 0.5
wr_score = _normalize(metrics.win_rate, low=0.50, high=0.65)
mwr_score = _normalize(metrics.monthly_win_rate, low=0.45, high=0.75)
consistency = wr_score * 0.5 + mwr_score * 0.5
r2_score = _normalize(metrics.equity_curve_r_squared, low=0.10, high=0.90)
tc_score = _normalize(float(metrics.trade_count), low=20.0, high=80.0)
robustness = r2_score * 0.5 + tc_score * 0.5
return_on_gross_exposure = None
if metrics.total_return_pct is not None and metrics.avg_gross_exposure_pct > 0:
return_on_gross_exposure = metrics.total_return_pct / metrics.avg_gross_exposure_pct
roe_score = _normalize(return_on_gross_exposure, low=0.05, high=0.40)
dim_score = _normalize_band(
metrics.days_in_market_pct,
low_bad=20.0,
low_good=45.0,
high_good=75.0,
high_bad=90.0,
)
capital_efficiency = roe_score * 0.7 + dim_score * 0.3
split_quality = (
profitability * w.split_profitability
+ risk * w.split_risk
+ consistency * w.split_consistency
+ robustness * w.split_robustness
+ capital_efficiency * w.split_capital_efficiency
)
if metrics.trade_count < 20:
split_quality *= 0.75
split_quality = round(split_quality, 1)
breakdown = {
"profitability": round(profitability, 1),
"risk": round(risk, 1),
"consistency": round(consistency, 1),
"robustness": round(robustness, 1),
"capital_efficiency": round(capital_efficiency, 1),
}
return split_quality, breakdown
def compute_promotion_score(
test_result: SplitResult | None,
valid_result: SplitResult | None,
weights: PromotionScoreWeights | None = None,
) -> tuple[float | None, dict[str, float]]:
"""Compute a promotion score using valid/test quality and a floor term."""
test_score, _, _ = _compute_split_quality_score(test_result)
valid_score, _, _ = _compute_split_quality_score(valid_result)
if test_score is None or valid_score is None:
return None, {}
w = weights or _DEFAULT_PROMOTION_WEIGHTS
floor_score = min(test_score, valid_score)
promotion_score = (
valid_score * w.valid_quality
+ test_score * w.test_quality
+ floor_score * w.floor_quality
)
breakdown = {
"valid_quality": round(valid_score, 1),
"test_quality": round(test_score, 1),
"floor_quality": round(floor_score, 1),
}
return round(promotion_score, 1), breakdown
def compute_unified_score(
test_result: SplitResult | None,
valid_result: SplitResult | None,
weights: UnifiedScoreWeights | None = None,
) -> tuple[float | None, dict[str, float]]:
"""Compute one integrated score for real promotion decisions."""
test_result = _hydrate_split_result(test_result)
valid_result = _hydrate_split_result(valid_result)
test_metrics = _metrics_from_split_result(test_result)
valid_metrics = _metrics_from_split_result(valid_result)
if test_metrics is None or valid_metrics is None:
return None, {}
w = weights or _DEFAULT_UNIFIED_WEIGHTS
test_quality, _ = compute_unified_split_quality(test_metrics, w)
valid_quality, _ = compute_unified_split_quality(valid_metrics, w)
if test_quality is None or valid_quality is None:
return None, {}
floor_quality = min(test_quality, valid_quality)
gap_quality = _normalize_inverse(abs(valid_quality - test_quality), low=35.0, high=5.0)
unified_score = (
valid_quality * w.valid_quality
+ test_quality * w.test_quality
+ floor_quality * w.floor_quality
+ gap_quality * w.gap_quality
)
breakdown = {
"valid_quality": round(valid_quality, 1),
"test_quality": round(test_quality, 1),
"floor_quality": round(floor_quality, 1),
"gap_quality": round(gap_quality, 1),
}
return _calibrate_sqs(unified_score), breakdown
def compute_public_sqs(
test_result: SplitResult | None,
valid_result: SplitResult | None,
) -> tuple[float | None, dict[str, float], str | None]:
"""Return the public-facing SQS.
Prefer the stricter integrated score when both valid/test are available.
Fall back to the best available single-split quality score otherwise,
using the same harsher calibration band.
"""
integrated_score, integrated_breakdown = compute_unified_score(test_result, valid_result)
if integrated_score is not None:
return integrated_score, integrated_breakdown, "integrated"
split_score, split_breakdown, split_source = _compute_split_quality_score(test_result)
if split_score is not None:
return _apply_single_split_penalty(_calibrate_sqs(split_score)), split_breakdown, split_source
return None, {}, None
def compute_config_delta(
current: dict[str, Any],
baseline: dict[str, Any],
baseline_name: str,
) -> ConfigDelta:
"""Compute a flat diff between two config dicts (one level deep)."""
changes: dict[str, str] = {}
_diff_recursive(baseline, current, prefix="", changes=changes)
return ConfigDelta(base_experiment=baseline_name, changes=changes)
def _diff_recursive(
old: dict[str, Any],
new: dict[str, Any],
prefix: str,
changes: dict[str, str],
) -> None:
all_keys = set(old.keys()) | set(new.keys())
for key in sorted(all_keys):
full_key = f"{prefix}{key}" if not prefix else f"{prefix}.{key}"
old_val = old.get(key)
new_val = new.get(key)
if isinstance(old_val, dict) and isinstance(new_val, dict):
_diff_recursive(old_val, new_val, full_key, changes)
elif old_val != new_val:
changes[full_key] = f"{old_val} \u2192 {new_val}"
# ---------------------------------------------------------------------------
# Journal I/O
# ---------------------------------------------------------------------------
def get_next_entry_id(journal_path: Path) -> str:
"""Return next entry ID like 'IMP-0001'."""
entries = load_journal(journal_path)
return f"IMP-{len(entries) + 1:04d}"
@contextlib.contextmanager
def journal_lock(journal_path: Path) -> Iterator[None]:
"""Serialize journal mutations across concurrent record commands."""
journal_path.parent.mkdir(parents=True, exist_ok=True)
lock_path = journal_path.with_suffix(f"{journal_path.suffix}.lock")
with lock_path.open("a+") as lock_file:
fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
try:
yield
finally:
fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
def append_journal_entry(journal_path: Path, entry: JournalEntry) -> None:
"""Append a single JournalEntry as one JSON line."""
journal_path.parent.mkdir(parents=True, exist_ok=True)
with open(journal_path, "a") as f:
f.write(entry.model_dump_json() + "\n")
logger.info("journal_entry_appended", entry_id=entry.entry_id, experiment=entry.experiment_name)
def load_journal(journal_path: Path) -> list[JournalEntry]:
"""Load all journal entries from JSONL file."""
if not journal_path.exists():
return []
entries: list[JournalEntry] = []
for line in journal_path.read_text().strip().splitlines():
line = line.strip()
if line:
entries.append(JournalEntry.model_validate_json(line))
return entries
# ---------------------------------------------------------------------------
# Registry / Leaderboard
# ---------------------------------------------------------------------------
def rebuild_registry(
journal_path: Path,
registry_path: Path,
leaderboard_path: Path,
) -> ExperimentRegistry:
"""Rebuild experiment_registry.json and LEADERBOARD.md from journal."""
entries = load_journal(journal_path)
registry_entries: list[RegistryEntry] = []
for je in entries:
test_result = _hydrate_split_result(je.results.get("test"))
valid_result = _hydrate_split_result(je.results.get("valid"))
computed_sqs_v2 = None
test_metrics = _metrics_from_split_result(test_result)
if test_metrics is not None:
computed_sqs_v2, _ = compute_sqs_v2(test_metrics)
computed_promotion_score, _ = compute_promotion_score(test_result, valid_result)
computed_unified_score, _ = compute_unified_score(test_result, valid_result)
computed_public_sqs, _, _ = compute_public_sqs(test_result, valid_result)
canonical_sqs = computed_public_sqs if computed_public_sqs is not None else je.sqs_score or 0.0
registry_entries.append(
RegistryEntry(
entry_id=je.entry_id,
experiment_name=je.experiment_name,
sqs_score=canonical_sqs,
sqs_v2_score=je.sqs_v2_score if je.sqs_v2_score is not None else computed_sqs_v2,
promotion_score=(
je.promotion_score if je.promotion_score is not None else computed_promotion_score
),
unified_score=(
je.unified_score if je.unified_score is not None else computed_unified_score
),
profit_factor=test_result.profit_factor if test_result else None,
total_return_pct=test_result.total_return_pct if test_result else None,
win_rate=test_result.win_rate if test_result else None,
sharpe_ratio=test_result.sharpe_ratio if test_result else None,
max_drawdown_pct=test_result.max_drawdown_pct if test_result else None,
trade_count=test_result.trade_count if test_result else 0,
avg_gross_exposure_pct=test_result.avg_gross_exposure_pct if test_result else None,
avg_net_exposure_pct=test_result.avg_net_exposure_pct if test_result else None,
days_in_market_pct=test_result.days_in_market_pct if test_result else None,
valid_profit_factor=valid_result.profit_factor if valid_result else None,
valid_total_return_pct=valid_result.total_return_pct if valid_result else None,
valid_win_rate=valid_result.win_rate if valid_result else None,
valid_sharpe_ratio=valid_result.sharpe_ratio if valid_result else None,
valid_max_drawdown_pct=valid_result.max_drawdown_pct if valid_result else None,
valid_trade_count=valid_result.trade_count if valid_result else 0,
valid_avg_gross_exposure_pct=valid_result.avg_gross_exposure_pct if valid_result else None,
valid_avg_net_exposure_pct=valid_result.avg_net_exposure_pct if valid_result else None,
valid_days_in_market_pct=valid_result.days_in_market_pct if valid_result else None,
timestamp=je.timestamp,
)
)
# Sort by canonical SQS descending, then promotion.
registry_entries.sort(
key=lambda e: (
-(e.sqs_score or 0.0),
e.promotion_score is None,
-(e.promotion_score or 0.0),
)
)
registry = ExperimentRegistry(
entries=registry_entries,
updated_at=utc_now().isoformat(),
)
# Write registry JSON
registry_path.parent.mkdir(parents=True, exist_ok=True)
registry_path.write_text(registry.model_dump_json(indent=2))
# Write LEADERBOARD.md
_write_leaderboard_md(leaderboard_path, registry, entries)
logger.info("registry_rebuilt", count=len(registry_entries))
return registry
def _write_leaderboard_md(
path: Path,
registry: ExperimentRegistry,
journal_entries: list[JournalEntry],
) -> None:
lines: list[str] = []
lines.append("# Strategy Improvement Leaderboard")
lines.append(f"_Updated: {registry.updated_at}_\n")
lines.append("| # | Experiment | SQS | [T]PF | [T]Ret% | [T]WR | [T]Sharpe | [T]DD% | [T]N | [T]Gross% | [T]Net% | [T]DIM% | [V]PF | [V]Ret% | [V]WR | [V]Sharpe | [V]DD% | [V]N | [V]Gross% | [V]Net% | [V]DIM% | Date |")
lines.append("|---|-----------|-----|-------|---------|-------|-----------|--------|------|-----------|---------|---------|-------|---------|-------|-----------|--------|------|-----------|---------|---------|------|")
for rank, e in enumerate(registry.entries, 1):
pf = f"{e.profit_factor:.2f}" if e.profit_factor is not None else "-"
ret = f"{e.total_return_pct:+.1f}" if e.total_return_pct is not None else "-"
wr = f"{e.win_rate:.0%}" if e.win_rate is not None else "-"
sharpe = f"{e.sharpe_ratio:.1f}" if e.sharpe_ratio is not None else "-"
dd = f"{e.max_drawdown_pct:.1f}" if e.max_drawdown_pct is not None else "-"
tgross = f"{e.avg_gross_exposure_pct:.1f}" if e.avg_gross_exposure_pct is not None else "-"
tnet = f"{e.avg_net_exposure_pct:+.1f}" if e.avg_net_exposure_pct is not None else "-"
tdim = f"{e.days_in_market_pct:.1f}" if e.days_in_market_pct is not None else "-"
vpf = f"{e.valid_profit_factor:.2f}" if e.valid_profit_factor is not None else "-"
vret = f"{e.valid_total_return_pct:+.1f}" if e.valid_total_return_pct is not None else "-"
vwr = f"{e.valid_win_rate:.0%}" if e.valid_win_rate is not None else "-"
vsharpe = f"{e.valid_sharpe_ratio:.1f}" if e.valid_sharpe_ratio is not None else "-"
vdd = f"{e.valid_max_drawdown_pct:.1f}" if e.valid_max_drawdown_pct is not None else "-"
vgross = f"{e.valid_avg_gross_exposure_pct:.1f}" if e.valid_avg_gross_exposure_pct is not None else "-"
vnet = f"{e.valid_avg_net_exposure_pct:+.1f}" if e.valid_avg_net_exposure_pct is not None else "-"
vdim = f"{e.valid_days_in_market_pct:.1f}" if e.valid_days_in_market_pct is not None else "-"
ts = e.timestamp[:10] if e.timestamp else "-"
lines.append(
f"| {rank} | {e.experiment_name} | {e.sqs_score:.1f}"
f" | {pf} | {ret} | {wr} | {sharpe} | {dd} | {e.trade_count} | {tgross} | {tnet} | {tdim}"
f" | {vpf} | {vret} | {vwr} | {vsharpe} | {vdd} | {e.valid_trade_count} | {vgross} | {vnet} | {vdim}"
f" | {ts} |"
)
# Recent entries (last 5)
recent = list(reversed(journal_entries))[:5]
if recent:
registry_by_id = {entry.entry_id: entry for entry in registry.entries}
lines.append("\n## Recent Entries")
for je in recent:
canonical_sqs = registry_by_id.get(je.entry_id).sqs_score if je.entry_id in registry_by_id else je.sqs_score
lines.append(f"### {je.entry_id} ({je.timestamp[:10]}) \u2014 {je.experiment_name}")
lines.append(f"Hypothesis: {je.hypothesis}")
lines.append(f"Verdict: **{je.verdict.upper()}** (SQS {canonical_sqs})")
if je.verdict_reasoning:
lines.append(f"Reasoning: {je.verdict_reasoning}")
if je.next_direction:
lines.append(f"Next: {je.next_direction}")
lines.append("")
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("\n".join(lines) + "\n")
# ---------------------------------------------------------------------------
# Run scanning helpers (for CLI record command)
# ---------------------------------------------------------------------------
def scan_runs_for_experiment(
runs_dir: Path,
experiment_name: str,
) -> dict[str, tuple[str, MetricsBundle]]:
"""Scan runs_dir for runs matching experiment_name.
Returns dict of split_name -> (run_id, MetricsBundle).
Uses manifest.json for experiment matching, metadata.json for split_name,
falls back to time-ordered grouping by config hash.
"""
from libs.backtest.domain import MetricsBundle
matches: list[tuple[Path, str, str]] = [] # (run_dir, run_id, config_hash)
for run_path in sorted(runs_dir.iterdir()):
if not run_path.is_dir():
continue
manifest_file = run_path / "manifest.json"
if not manifest_file.exists():
continue
manifest_data = json.loads(manifest_file.read_text())
if manifest_data.get("experiment_name") != experiment_name:
continue
metadata_file = run_path / "metadata.json"
metadata = json.loads(metadata_file.read_text()) if metadata_file.exists() else {}
run_id = metadata.get("run_id", run_path.name)
# Extract config hash from run_id (last segment after underscore)
parts = run_path.name.split("_")
config_hash = parts[-1] if len(parts) >= 2 else ""
matches.append((run_path, run_id, config_hash))
if not matches:
return {}
results: dict[str, tuple[str, MetricsBundle]] = {}
# Check for split_name in metadata first
has_split_names = False
for run_path, run_id, _ in matches:
metadata_file = run_path / "metadata.json"
if metadata_file.exists():
metadata = json.loads(metadata_file.read_text())
if "split_name" in metadata:
has_split_names = True
break
if has_split_names:
for run_path, run_id, _ in matches:
metadata = json.loads((run_path / "metadata.json").read_text())
split = metadata.get("split_name", "unknown")
metrics_file = run_path / "metrics" / "metrics_summary.json"
if metrics_file.exists():
metrics = MetricsBundle.model_validate_json(metrics_file.read_text())
results[split] = (run_id, metrics)
else:
# Fallback: group by config_hash, then assign train/valid/test by time order
from collections import defaultdict
groups: dict[str, list[tuple[Path, str]]] = defaultdict(list)
for run_path, run_id, config_hash in matches:
groups[config_hash].append((run_path, run_id))
# Use the largest group (most likely the 3-split set)
biggest_group = max(groups.values(), key=len) if groups else []
split_names = ["train", "valid", "test"]
for i, (run_path, run_id) in enumerate(biggest_group):
split = split_names[i] if i < len(split_names) else f"extra_{i}"
metrics_file = run_path / "metrics" / "metrics_summary.json"
if metrics_file.exists():
metrics = MetricsBundle.model_validate_json(metrics_file.read_text())
results[split] = (run_id, metrics)
return results
def check_duplicate(journal_path: Path, experiment_name: str) -> list[JournalEntry]:
"""Check if experiment_name already exists in journal."""
entries = load_journal(journal_path)
return [e for e in entries if e.experiment_name == experiment_name]
Reference in New Issue View Git Blame Copy Permalink