You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1500 lines
62 KiB
Python
1500 lines
62 KiB
Python
"""CLI for strategy improvement tracking: record, leaderboard, check-duplicate."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from rich import box
|
|
from rich.console import Console
|
|
from rich.panel import Panel
|
|
from rich.table import Table
|
|
|
|
from libs.backtest.domain import (
|
|
ConfigDelta,
|
|
JournalEntry,
|
|
ResetCommonWindowSummary,
|
|
RobustnessMatrixSummary,
|
|
SplitResult,
|
|
WalkForwardSummary,
|
|
)
|
|
from libs.backtest.tracker import (
|
|
_hydrate_split_result,
|
|
append_journal_entry,
|
|
attach_out_of_time_robustness_summary,
|
|
attach_reset_common_window_summary,
|
|
attach_robustness_summary,
|
|
attach_walk_forward_summary,
|
|
build_split_result,
|
|
check_duplicate,
|
|
compute_deployment_score,
|
|
compute_public_sqs,
|
|
compute_public_sqs_v2,
|
|
compute_promotion_score,
|
|
compute_oot_robustness_gate,
|
|
compute_reset_common_window_score,
|
|
compute_robustness_gate,
|
|
compute_rqs,
|
|
compute_sqs,
|
|
compute_sqs_v2,
|
|
compute_unified_score,
|
|
compute_wfqs,
|
|
compute_wfqs_v2,
|
|
filter_registry_entries,
|
|
get_next_entry_id,
|
|
journal_lock,
|
|
load_journal,
|
|
refresh_public_scores,
|
|
rebuild_registry,
|
|
scan_runs_for_experiment,
|
|
sync_official_manifests,
|
|
)
|
|
from libs.common.time_utils import utc_now
|
|
|
|
_console = Console(width=140)
|
|
_DEFAULT_JOURNAL_DIR = "journal"
|
|
_COL_NAME = 38
|
|
|
|
|
|
def _fmt(val: float | None, fmt: str) -> str:
|
|
return format(val, fmt) if val is not None else "-"
|
|
|
|
|
|
def _ratio(numerator: float | None, denominator: float | None) -> float | None:
|
|
if numerator is None or denominator is None or denominator == 0:
|
|
return None
|
|
return numerator / denominator
|
|
|
|
|
|
def _entry_matches_target(entry: JournalEntry, target: str) -> bool:
|
|
target_upper = target.upper()
|
|
return entry.entry_id == target_upper or entry.experiment_name == target
|
|
|
|
|
|
def _ordered_splits(results: dict[str, SplitResult]) -> list[tuple[str, SplitResult]]:
|
|
preferred = ["train", "valid", "test"]
|
|
ordered: list[tuple[str, SplitResult]] = []
|
|
seen: set[str] = set()
|
|
for name in preferred:
|
|
if name in results:
|
|
ordered.append((name, results[name]))
|
|
seen.add(name)
|
|
for name in sorted(results.keys()):
|
|
if name not in seen:
|
|
ordered.append((name, results[name]))
|
|
return ordered
|
|
|
|
|
|
def _sync_and_rebuild(
|
|
journal_path: Path,
|
|
registry_path: Path,
|
|
leaderboard_path: Path,
|
|
) -> object:
|
|
sync_official_manifests(journal_path, Path("runs"))
|
|
return rebuild_registry(journal_path, registry_path, leaderboard_path)
|
|
|
|
|
|
def _load_optional_wfv_summary(path_str: str | None) -> WalkForwardSummary | None:
|
|
if not path_str:
|
|
return None
|
|
path = Path(path_str)
|
|
if not path.exists():
|
|
print(f"ERROR: walk-forward summary not found: {path}")
|
|
sys.exit(1)
|
|
return WalkForwardSummary.model_validate_json(path.read_text())
|
|
|
|
|
|
def _load_optional_robustness_summary(path_str: str | None) -> RobustnessMatrixSummary | None:
|
|
if not path_str:
|
|
return None
|
|
path = Path(path_str)
|
|
if not path.exists():
|
|
print(f"ERROR: robustness summary not found: {path}")
|
|
sys.exit(1)
|
|
return RobustnessMatrixSummary.model_validate_json(path.read_text())
|
|
|
|
|
|
def _load_optional_reset_common_window_summary(
|
|
path_str: str | None,
|
|
) -> ResetCommonWindowSummary | None:
|
|
if not path_str:
|
|
return None
|
|
path = Path(path_str)
|
|
if not path.exists():
|
|
print(f"ERROR: reset common-window summary not found: {path}")
|
|
sys.exit(1)
|
|
return ResetCommonWindowSummary.model_validate_json(path.read_text())
|
|
|
|
|
|
def _score_sort_key(sort_by: str, entry) -> tuple:
|
|
if sort_by == "promotion":
|
|
return (
|
|
entry.promotion_score is None,
|
|
-(entry.promotion_score or 0.0),
|
|
-(entry.sqs_score or 0.0),
|
|
)
|
|
if sort_by in {"deployment", "dep"}:
|
|
return (
|
|
entry.deployment_score is None,
|
|
-(entry.deployment_score or 0.0),
|
|
entry.wfqs_score is None,
|
|
-(entry.wfqs_score or 0.0),
|
|
-(entry.rqs_score or 0.0),
|
|
-(entry.sqs_score or 0.0),
|
|
)
|
|
if sort_by == "wfqs":
|
|
return (
|
|
entry.wfqs_score is None,
|
|
-(entry.wfqs_score or 0.0),
|
|
-(entry.deployment_score or 0.0),
|
|
-(entry.rqs_score or 0.0),
|
|
-(entry.sqs_score or 0.0),
|
|
)
|
|
if sort_by == "rqs":
|
|
return (
|
|
entry.rqs_score is None,
|
|
-(entry.rqs_score or 0.0),
|
|
-(entry.deployment_score or 0.0),
|
|
-(entry.sqs_score or 0.0),
|
|
)
|
|
if sort_by == "unified":
|
|
return (
|
|
entry.unified_score is None,
|
|
-(entry.unified_score or 0.0),
|
|
-(entry.sqs_score or 0.0),
|
|
)
|
|
if sort_by == "sqs2":
|
|
return (
|
|
entry.wfqs_v2_score is None,
|
|
-(entry.wfqs_v2_score or 0.0),
|
|
-(entry.wfqs_score or 0.0),
|
|
-(entry.sqs_score or 0.0),
|
|
)
|
|
return (-(entry.sqs_score or 0.0),)
|
|
|
|
|
|
def _validation_pending_reason(entry: JournalEntry) -> str | None:
|
|
missing: list[str] = []
|
|
if entry.walk_forward_summary is None:
|
|
missing.append("WFV")
|
|
if entry.robustness_matrix_summary is None:
|
|
missing.append("robustness")
|
|
if entry.out_of_time_robustness_summary is None:
|
|
missing.append("OOT robustness")
|
|
if not missing:
|
|
return None
|
|
return "missing " + " + ".join(missing)
|
|
|
|
|
|
def _title_for_sort(sort_by: str) -> str:
|
|
if sort_by == "promotion":
|
|
return "Promotion 기준 내림차순"
|
|
if sort_by in {"deployment", "dep"}:
|
|
return "Deployment 기준 내림차순"
|
|
if sort_by == "wfqs":
|
|
return "WFQS 기준 내림차순"
|
|
if sort_by == "rqs":
|
|
return "RQS 기준 내림차순"
|
|
if sort_by == "unified":
|
|
return "Unified 기준 내림차순"
|
|
if sort_by == "sqs2":
|
|
return "WFQS v2 기준 내림차순"
|
|
return "SQS 기준 내림차순"
|
|
|
|
|
|
def cmd_record(args: argparse.Namespace) -> None:
|
|
"""Record an experiment to the improvement journal."""
|
|
journal_dir = Path(args.journal_dir)
|
|
journal_path = journal_dir / "improvement_journal.jsonl"
|
|
registry_path = journal_dir / "experiment_registry.json"
|
|
leaderboard_path = journal_dir / "LEADERBOARD.md"
|
|
|
|
runs_dir = Path(args.runs_dir)
|
|
if not runs_dir.exists():
|
|
print(f"ERROR: runs directory not found: {runs_dir}")
|
|
sys.exit(1)
|
|
|
|
split_runs = scan_runs_for_experiment(runs_dir, args.experiment)
|
|
if not split_runs:
|
|
print(f"ERROR: no runs found for experiment '{args.experiment}' in {runs_dir}")
|
|
sys.exit(1)
|
|
|
|
results: dict[str, SplitResult] = {}
|
|
for split_name, (run_id, metrics) in split_runs.items():
|
|
results[split_name] = build_split_result(split_name, run_id, metrics)
|
|
|
|
test_metrics = None
|
|
for preferred in ["test", "valid", "train"]:
|
|
if preferred in split_runs:
|
|
_, test_metrics = split_runs[preferred]
|
|
break
|
|
|
|
sqs_score = None
|
|
sqs_breakdown: dict[str, float] = {}
|
|
sqs_v2_score = None
|
|
sqs_v2_breakdown: dict[str, float] = {}
|
|
if test_metrics is not None:
|
|
sqs_score, sqs_breakdown = compute_sqs(test_metrics)
|
|
sqs_v2_score, sqs_v2_breakdown = compute_sqs_v2(test_metrics)
|
|
|
|
promotion_score, promotion_breakdown = compute_promotion_score(
|
|
results.get("test"),
|
|
results.get("valid"),
|
|
)
|
|
unified_score, unified_breakdown = compute_unified_score(
|
|
results.get("test"),
|
|
results.get("valid"),
|
|
)
|
|
rqs_score, rqs_breakdown = compute_rqs(
|
|
results.get("train"),
|
|
results.get("valid"),
|
|
results.get("test"),
|
|
)
|
|
|
|
walk_forward_summary = _load_optional_wfv_summary(args.walk_forward_summary)
|
|
robustness_matrix_summary = _load_optional_robustness_summary(args.robustness_summary)
|
|
out_of_time_robustness_summary = _load_optional_robustness_summary(
|
|
getattr(args, "out_of_time_robustness_summary", None)
|
|
)
|
|
reset_common_window_summary = _load_optional_reset_common_window_summary(
|
|
getattr(args, "reset_common_window_summary", None)
|
|
)
|
|
|
|
wfqs_score = None
|
|
wfqs_breakdown: dict[str, float] = {}
|
|
wfqs_v2_score = None
|
|
wfqs_v2_breakdown: dict[str, float] = {}
|
|
deployment_score = None
|
|
deployment_breakdown: dict[str, float] = {}
|
|
if walk_forward_summary is not None:
|
|
wfqs_score, wfqs_breakdown = compute_wfqs(walk_forward_summary)
|
|
wfqs_v2_score, wfqs_v2_breakdown = compute_wfqs_v2(walk_forward_summary)
|
|
deployment_score, deployment_breakdown = compute_deployment_score(
|
|
results.get("train"),
|
|
results.get("valid"),
|
|
results.get("test"),
|
|
walk_forward_summary,
|
|
rqs_score=rqs_score,
|
|
wfqs_score=wfqs_score,
|
|
)
|
|
|
|
public_sqs, public_breakdown, public_source = compute_public_sqs(
|
|
results.get("train"),
|
|
results.get("valid"),
|
|
results.get("test"),
|
|
walk_forward_summary=walk_forward_summary,
|
|
robustness_matrix_summary=robustness_matrix_summary,
|
|
out_of_time_robustness_summary=out_of_time_robustness_summary,
|
|
reset_common_window_summary=reset_common_window_summary,
|
|
rqs_score=rqs_score,
|
|
wfqs_score=wfqs_v2_score,
|
|
)
|
|
stress_sqs, stress_breakdown, _ = compute_public_sqs_v2(
|
|
results.get("train"),
|
|
results.get("valid"),
|
|
results.get("test"),
|
|
walk_forward_summary=walk_forward_summary,
|
|
robustness_matrix_summary=robustness_matrix_summary,
|
|
out_of_time_robustness_summary=out_of_time_robustness_summary,
|
|
rqs_score=rqs_score,
|
|
wfqs_v2_score=wfqs_v2_score,
|
|
)
|
|
if public_sqs is not None:
|
|
sqs_score = public_sqs
|
|
sqs_breakdown = public_breakdown
|
|
|
|
reset_common_window_score = None
|
|
reset_common_window_breakdown: dict[str, float] = {}
|
|
if reset_common_window_summary is not None:
|
|
reset_common_window_score, reset_common_window_breakdown = compute_reset_common_window_score(
|
|
reset_common_window_summary
|
|
)
|
|
|
|
config_delta = None
|
|
if args.baseline:
|
|
config_delta = ConfigDelta(base_experiment=args.baseline, changes={})
|
|
|
|
tags = [tag for tag in args.experiment.replace("-", "_").split("_") if tag]
|
|
|
|
with journal_lock(journal_path):
|
|
dupes = check_duplicate(journal_path, args.experiment)
|
|
if dupes and not args.force:
|
|
print(f"WARNING: experiment '{args.experiment}' already in journal ({len(dupes)} entries).")
|
|
print("Use --force to add anyway.")
|
|
sys.exit(1)
|
|
|
|
entry_id = get_next_entry_id(journal_path)
|
|
entry = JournalEntry(
|
|
entry_id=entry_id,
|
|
timestamp=utc_now().isoformat(),
|
|
experiment_name=args.experiment,
|
|
hypothesis=args.hypothesis or "",
|
|
config_delta=config_delta,
|
|
results=results,
|
|
walk_forward_summary=walk_forward_summary,
|
|
robustness_matrix_summary=robustness_matrix_summary,
|
|
out_of_time_robustness_summary=out_of_time_robustness_summary,
|
|
reset_common_window_summary=reset_common_window_summary,
|
|
sqs_score=sqs_score,
|
|
sqs_breakdown=sqs_breakdown,
|
|
stress_sqs_score=stress_sqs,
|
|
stress_sqs_breakdown=stress_breakdown,
|
|
sqs_v2_score=sqs_v2_score,
|
|
sqs_v2_breakdown=sqs_v2_breakdown,
|
|
promotion_score=promotion_score,
|
|
promotion_breakdown=promotion_breakdown,
|
|
unified_score=unified_score,
|
|
unified_breakdown=unified_breakdown,
|
|
rqs_score=rqs_score,
|
|
rqs_breakdown=rqs_breakdown,
|
|
wfqs_score=wfqs_score,
|
|
wfqs_breakdown=wfqs_breakdown,
|
|
wfqs_v2_score=wfqs_v2_score,
|
|
wfqs_v2_breakdown=wfqs_v2_breakdown,
|
|
deployment_score=deployment_score,
|
|
deployment_breakdown=deployment_breakdown,
|
|
reset_common_window_score=reset_common_window_score,
|
|
reset_common_window_breakdown=reset_common_window_breakdown,
|
|
verdict=args.verdict or "unknown",
|
|
verdict_reasoning=args.reasoning or "",
|
|
next_direction=args.next or "",
|
|
tags=tags,
|
|
)
|
|
|
|
append_journal_entry(journal_path, entry)
|
|
_sync_and_rebuild(journal_path, registry_path, leaderboard_path)
|
|
|
|
if sqs_score is None:
|
|
print(f"Recorded {entry_id}: {args.experiment} (SQS=pending validation)")
|
|
else:
|
|
source_label = f", source={public_source}" if public_source else ""
|
|
stress_label = f", stress={stress_sqs:.1f}" if stress_sqs is not None else ""
|
|
print(f"Recorded {entry_id}: {args.experiment} (SQS={sqs_score}{stress_label}{source_label})")
|
|
for split_name, sr in _ordered_splits(results):
|
|
pf = f"PF={sr.profit_factor:.2f}" if sr.profit_factor is not None else "PF=-"
|
|
ret = f"Ret={sr.total_return_pct:+.2f}%" if sr.total_return_pct is not None else "Ret=-"
|
|
print(f" {split_name}: {sr.trade_count} trades, {pf}, {ret}")
|
|
print(f"Leaderboard updated: {leaderboard_path}")
|
|
|
|
|
|
def cmd_leaderboard(args: argparse.Namespace) -> None:
|
|
"""Show or regenerate the leaderboard."""
|
|
journal_dir = Path(args.journal_dir)
|
|
journal_path = journal_dir / "improvement_journal.jsonl"
|
|
registry_path = journal_dir / "experiment_registry.json"
|
|
leaderboard_path = journal_dir / "LEADERBOARD.md"
|
|
|
|
if not journal_path.exists():
|
|
journal_path.parent.mkdir(parents=True, exist_ok=True)
|
|
journal_path.touch()
|
|
|
|
with journal_lock(journal_path):
|
|
registry = _sync_and_rebuild(journal_path, registry_path, leaderboard_path)
|
|
ranked_source = filter_registry_entries(
|
|
registry.entries,
|
|
include_retired=getattr(args, "include_retired", False),
|
|
)
|
|
displayed_leaderboard_path = leaderboard_path
|
|
|
|
sort_by = getattr(args, "sort", "sqs")
|
|
ranked_entries = sorted(ranked_source, key=lambda entry: _score_sort_key(sort_by, entry))
|
|
total = len(ranked_entries)
|
|
top_n = getattr(args, "top", 10)
|
|
diagnostics = sort_by in {"deployment", "dep", "wfqs", "rqs", "promotion", "unified", "sqs2"}
|
|
title = f"[bold cyan]Top {top_n} / {total}[/] [dim]· {_title_for_sort(sort_by)} · Tr=train V=valid T=test[/]"
|
|
|
|
tbl = Table(
|
|
box=box.SIMPLE_HEAD,
|
|
show_header=True,
|
|
header_style="bold yellow",
|
|
row_styles=["", "dim"],
|
|
padding=(0, 1),
|
|
title=title,
|
|
title_justify="left",
|
|
expand=False,
|
|
)
|
|
# Load experiment IDs from index for display
|
|
_exp_id_map: dict[str, str] = {}
|
|
try:
|
|
import json as _json
|
|
_idx = Path("configs/experiments/.index.json")
|
|
if _idx.exists():
|
|
_idx_data = _json.loads(_idx.read_text())
|
|
for _ename, _emeta in _idx_data.get("experiments", {}).items():
|
|
eid = _emeta.get("id")
|
|
if eid is not None:
|
|
_exp_id_map[_ename] = str(eid)
|
|
except Exception:
|
|
pass
|
|
|
|
tbl.add_column("ID", justify="right", style="bold dim", no_wrap=True, min_width=4)
|
|
tbl.add_column("#", justify="right", style="bold", no_wrap=True, min_width=3)
|
|
tbl.add_column("Experiment", no_wrap=True, min_width=30)
|
|
tbl.add_column("SQS", justify="right", style="bold cyan", no_wrap=True, min_width=5)
|
|
tbl.add_column("RCW", justify="right", style="bright_magenta", no_wrap=True, min_width=5)
|
|
if diagnostics:
|
|
if sort_by == "promotion":
|
|
tbl.add_column("Promo", justify="right", style="bright_magenta", no_wrap=True, min_width=5)
|
|
else:
|
|
tbl.add_column("DEP", justify="right", style="bold green", no_wrap=True, min_width=5)
|
|
tbl.add_column("WFQS", justify="right", style="bright_cyan", no_wrap=True, min_width=5)
|
|
tbl.add_column("RQS", justify="right", style="bold magenta", no_wrap=True, min_width=5)
|
|
tbl.add_column("Tr.Ret%", justify="right", style="bright_blue", no_wrap=True, min_width=7)
|
|
tbl.add_column("V.Ret%", justify="right", style="green", no_wrap=True, min_width=6)
|
|
tbl.add_column("T.Ret%", justify="right", no_wrap=True, min_width=6)
|
|
tbl.add_column("T.Ann%", justify="right", no_wrap=True, min_width=7)
|
|
tbl.add_column("T.DD%", justify="right", no_wrap=True, min_width=5)
|
|
tbl.add_column("T.Gross%", justify="right", no_wrap=True, min_width=8)
|
|
tbl.add_column("T.DIM%", justify="right", no_wrap=True, min_width=6)
|
|
tbl.add_column("T.R/G", justify="right", no_wrap=True, min_width=5)
|
|
|
|
for rank, entry in enumerate(ranked_entries[:top_n], 1):
|
|
name = entry.experiment_name
|
|
if len(name) > _COL_NAME:
|
|
name = name[: _COL_NAME - 1] + "…"
|
|
row = [
|
|
_exp_id_map.get(entry.experiment_name, "—"),
|
|
str(rank),
|
|
name,
|
|
_fmt(entry.sqs_score, ".1f"),
|
|
_fmt(entry.reset_common_window_score, ".1f"),
|
|
]
|
|
if diagnostics:
|
|
if sort_by == "promotion":
|
|
row.append(_fmt(entry.promotion_score, ".1f"))
|
|
else:
|
|
row.extend([
|
|
_fmt(entry.deployment_score, ".1f"),
|
|
_fmt(entry.wfqs_score, ".1f"),
|
|
_fmt(entry.rqs_score, ".1f"),
|
|
])
|
|
row.extend([
|
|
_fmt(entry.train_total_return_pct, "+.1f"),
|
|
_fmt(entry.valid_total_return_pct, "+.1f"),
|
|
_fmt(entry.total_return_pct, "+.1f"),
|
|
_fmt(entry.annualized_return_pct, "+.1f"),
|
|
_fmt(entry.max_drawdown_pct, ".1f"),
|
|
_fmt(entry.avg_gross_exposure_pct, ".1f"),
|
|
_fmt(entry.days_in_market_pct, ".1f"),
|
|
_fmt(_ratio(entry.total_return_pct, entry.avg_gross_exposure_pct), ".2f"),
|
|
])
|
|
tbl.add_row(*row)
|
|
|
|
_console.print()
|
|
_console.print(tbl)
|
|
_console.print(f" [dim]LEADERBOARD.md → {displayed_leaderboard_path}[/]\n")
|
|
|
|
|
|
def cmd_show(args: argparse.Namespace) -> None:
|
|
"""Show details of a specific journal entry."""
|
|
journal_dir = Path(args.journal_dir)
|
|
journal_path = journal_dir / "improvement_journal.jsonl"
|
|
registry_path = journal_dir / "experiment_registry.json"
|
|
leaderboard_path = journal_dir / "LEADERBOARD.md"
|
|
with journal_lock(journal_path):
|
|
_sync_and_rebuild(journal_path, registry_path, leaderboard_path)
|
|
entries = load_journal(journal_path)
|
|
|
|
target = args.entry_id
|
|
found = [entry for entry in entries if _entry_matches_target(entry, target)]
|
|
if not found:
|
|
found = [entry for entry in entries if target.lower() in entry.experiment_name.lower()]
|
|
if not found:
|
|
print(f"No entry found for: {target}")
|
|
sys.exit(1)
|
|
|
|
for entry in found:
|
|
train_result = _hydrate_split_result(entry.results.get("train"))
|
|
valid_result = _hydrate_split_result(entry.results.get("valid"))
|
|
test_result = _hydrate_split_result(entry.results.get("test"))
|
|
fresh_rqs_score, fresh_rqs_breakdown = compute_rqs(
|
|
train_result,
|
|
valid_result,
|
|
test_result,
|
|
)
|
|
fresh_wfqs_score, fresh_wfqs_breakdown = compute_wfqs(entry.walk_forward_summary)
|
|
fresh_wfqs_v2_score, _ = compute_wfqs_v2(entry.walk_forward_summary)
|
|
fresh_deployment_score, fresh_deployment_breakdown = compute_deployment_score(
|
|
train_result,
|
|
valid_result,
|
|
test_result,
|
|
entry.walk_forward_summary,
|
|
rqs_score=fresh_rqs_score,
|
|
wfqs_score=fresh_wfqs_score,
|
|
)
|
|
public_sqs, public_breakdown, public_source = compute_public_sqs(
|
|
train_result,
|
|
valid_result,
|
|
test_result,
|
|
walk_forward_summary=entry.walk_forward_summary,
|
|
robustness_matrix_summary=entry.robustness_matrix_summary,
|
|
out_of_time_robustness_summary=entry.out_of_time_robustness_summary,
|
|
common_window_summary=entry.common_window_summary,
|
|
reset_common_window_summary=entry.reset_common_window_summary,
|
|
multi_capital_common_window_summary=entry.multi_capital_common_window_summary,
|
|
rqs_score=fresh_rqs_score,
|
|
wfqs_score=fresh_wfqs_v2_score,
|
|
)
|
|
stress_sqs, _, _ = compute_public_sqs_v2(
|
|
train_result,
|
|
valid_result,
|
|
test_result,
|
|
walk_forward_summary=entry.walk_forward_summary,
|
|
robustness_matrix_summary=entry.robustness_matrix_summary,
|
|
out_of_time_robustness_summary=entry.out_of_time_robustness_summary,
|
|
rqs_score=fresh_rqs_score,
|
|
wfqs_v2_score=fresh_wfqs_v2_score,
|
|
)
|
|
if public_sqs is None:
|
|
public_breakdown = public_breakdown or {}
|
|
public_source = public_source or "pending_validation"
|
|
|
|
print(f"\n{entry.entry_id} — {entry.experiment_name}")
|
|
print(f" Timestamp: {entry.timestamp}")
|
|
print(f" Hypothesis: {entry.hypothesis}")
|
|
print(f" Verdict: {entry.verdict}")
|
|
pending_reason = _validation_pending_reason(entry)
|
|
if public_sqs is None:
|
|
print(f" SQS: pending [{public_source}]")
|
|
if pending_reason:
|
|
print(f" Validation: {pending_reason}")
|
|
else:
|
|
print(f" SQS: {public_sqs} [{public_source}]")
|
|
if stress_sqs is not None:
|
|
print(f" Stress SQS: {stress_sqs}")
|
|
|
|
if entry.config_delta:
|
|
print(f" Baseline: {entry.config_delta.base_experiment}")
|
|
for key, value in entry.config_delta.changes.items():
|
|
print(f" {key}: {value}")
|
|
|
|
hydrated_results = {
|
|
"train": train_result,
|
|
"valid": valid_result,
|
|
"test": test_result,
|
|
**{
|
|
name: result
|
|
for name, result in entry.results.items()
|
|
if name not in {"train", "valid", "test"}
|
|
},
|
|
}
|
|
for split_name, split_result in _ordered_splits(hydrated_results):
|
|
pf = f"PF={split_result.profit_factor:.2f}" if split_result.profit_factor is not None else "PF=-"
|
|
ret = f"Ret={split_result.total_return_pct:+.2f}%" if split_result.total_return_pct is not None else "Ret=-"
|
|
ann = (
|
|
f"Ann={split_result.annualized_return_pct:+.2f}%"
|
|
if split_result.annualized_return_pct is not None
|
|
else "Ann=-"
|
|
)
|
|
dd = f"DD={split_result.max_drawdown_pct:.2f}%" if split_result.max_drawdown_pct is not None else "DD=-"
|
|
gross = (
|
|
f"Gross={split_result.avg_gross_exposure_pct:.2f}%"
|
|
if split_result.avg_gross_exposure_pct is not None
|
|
else "Gross=-"
|
|
)
|
|
dim = (
|
|
f"DIM={split_result.days_in_market_pct:.2f}%"
|
|
if split_result.days_in_market_pct is not None
|
|
else "DIM=-"
|
|
)
|
|
ret_on_gross = _ratio(split_result.total_return_pct, split_result.avg_gross_exposure_pct)
|
|
rog = f"R/G={ret_on_gross:.2f}" if ret_on_gross is not None else "R/G=-"
|
|
print(f" {split_name}: {split_result.trade_count} trades, {pf}, {ret}, {ann}, {dd}, {gross}, {dim}, {rog}")
|
|
|
|
if entry.walk_forward_summary is not None:
|
|
summary = entry.walk_forward_summary
|
|
test = summary.test_aggregate
|
|
gap = summary.gap_stats
|
|
print(
|
|
" WFV: "
|
|
f"{summary.fold_count} folds, "
|
|
f"mean={_fmt(test.mean_return_pct, '+.2f')}%, "
|
|
f"median={_fmt(test.median_return_pct, '+.2f')}%, "
|
|
f"worst={_fmt(test.worst_return_pct, '+.2f')}%, "
|
|
f"positive={_fmt(test.positive_fold_rate_pct, '.1f')}%"
|
|
)
|
|
print(
|
|
" WFV Gap: "
|
|
f"mean train-test gap={_fmt(gap.mean_train_test_return_gap_pct, '.2f')}%, "
|
|
f"worst={_fmt(gap.worst_train_test_return_gap_pct, '.2f')}%"
|
|
)
|
|
|
|
if entry.robustness_matrix_summary is not None:
|
|
summary = entry.robustness_matrix_summary
|
|
h63 = next((item for item in summary.horizon_summaries if item.horizon_days == 63), None)
|
|
h252 = next((item for item in summary.horizon_summaries if item.horizon_days == 252), None)
|
|
print(
|
|
" Robust: "
|
|
f"windows={summary.overall_window_count}, "
|
|
f"positive={_fmt(summary.overall_positive_window_rate_pct, '.1f')}%, "
|
|
f"worst={_fmt(summary.overall_worst_return_pct, '+.2f')}%, "
|
|
f"63d med={_fmt(h63.median_return_pct if h63 else None, '+.2f')}%, "
|
|
f"252d med={_fmt(h252.median_return_pct if h252 else None, '+.2f')}%"
|
|
)
|
|
for horizon in summary.horizon_summaries:
|
|
print(
|
|
" "
|
|
f"{horizon.horizon_days:>3}d: "
|
|
f"n={horizon.window_count}, "
|
|
f"mean={_fmt(horizon.mean_return_pct, '+.2f')}%, "
|
|
f"median={_fmt(horizon.median_return_pct, '+.2f')}%, "
|
|
f"worst={_fmt(horizon.worst_return_pct, '+.2f')}%, "
|
|
f"positive={_fmt(horizon.positive_window_rate_pct, '.1f')}%, "
|
|
f"dd={_fmt(horizon.mean_max_drawdown_pct, '.2f')}%"
|
|
)
|
|
if entry.out_of_time_robustness_summary is not None:
|
|
summary = entry.out_of_time_robustness_summary
|
|
h63 = next((item for item in summary.horizon_summaries if item.horizon_days == 63), None)
|
|
h252 = next((item for item in summary.horizon_summaries if item.horizon_days == 252), None)
|
|
print(
|
|
" OOT Robust: "
|
|
f"windows={summary.overall_window_count}, "
|
|
f"positive={_fmt(summary.overall_positive_window_rate_pct, '.1f')}%, "
|
|
f"worst={_fmt(summary.overall_worst_return_pct, '+.2f')}%, "
|
|
f"63d med={_fmt(h63.median_return_pct if h63 else None, '+.2f')}%, "
|
|
f"252d med={_fmt(h252.median_return_pct if h252 else None, '+.2f')}%"
|
|
)
|
|
|
|
if args.diagnostics:
|
|
promotion_score = entry.promotion_score
|
|
promotion_breakdown = entry.promotion_breakdown
|
|
if promotion_score is None:
|
|
promotion_score, promotion_breakdown = compute_promotion_score(
|
|
test_result,
|
|
valid_result,
|
|
)
|
|
unified_score = entry.unified_score
|
|
unified_breakdown = entry.unified_breakdown
|
|
if unified_score is None:
|
|
unified_score, unified_breakdown = compute_unified_score(
|
|
test_result,
|
|
valid_result,
|
|
)
|
|
rqs_score = fresh_rqs_score
|
|
rqs_breakdown = fresh_rqs_breakdown
|
|
wfqs_score = fresh_wfqs_score
|
|
wfqs_breakdown = fresh_wfqs_breakdown
|
|
deployment_score = fresh_deployment_score
|
|
deployment_breakdown = fresh_deployment_breakdown
|
|
robustness_gate_factor, robustness_breakdown = compute_robustness_gate(
|
|
entry.robustness_matrix_summary,
|
|
)
|
|
oot_gate_factor, oot_breakdown = compute_oot_robustness_gate(
|
|
entry.out_of_time_robustness_summary,
|
|
)
|
|
print(" Diagnostics:")
|
|
print(f" Public SQS: {public_sqs} {public_breakdown}")
|
|
if rqs_score is not None:
|
|
print(f" RQS: {rqs_score} {rqs_breakdown}")
|
|
if wfqs_score is not None:
|
|
print(f" WFQS: {wfqs_score} {wfqs_breakdown}")
|
|
wfqs_v2_score = entry.wfqs_v2_score
|
|
wfqs_v2_breakdown = entry.wfqs_v2_breakdown
|
|
if wfqs_v2_score is None:
|
|
wfqs_v2_score, wfqs_v2_breakdown = compute_wfqs_v2(entry.walk_forward_summary)
|
|
if wfqs_v2_score is not None:
|
|
print(f" WFQS v2: {wfqs_v2_score} {wfqs_v2_breakdown}")
|
|
if deployment_score is not None:
|
|
print(f" Deploy: {deployment_score} {deployment_breakdown}")
|
|
if promotion_score is not None:
|
|
print(f" Promotion: {promotion_score} {promotion_breakdown}")
|
|
if unified_score is not None:
|
|
print(f" Unified: {unified_score} {unified_breakdown}")
|
|
if entry.robustness_matrix_summary is not None:
|
|
print(f" RobustGate: {robustness_gate_factor:.2f} {robustness_breakdown}")
|
|
if entry.out_of_time_robustness_summary is not None:
|
|
print(f" OOTGate: {oot_gate_factor:.2f} {oot_breakdown}")
|
|
|
|
if entry.verdict_reasoning:
|
|
print(f" Reasoning: {entry.verdict_reasoning}")
|
|
if entry.next_direction:
|
|
print(f" Next: {entry.next_direction}")
|
|
|
|
|
|
def cmd_check_duplicate(args: argparse.Namespace) -> None:
|
|
"""Check if an experiment has been recorded already."""
|
|
journal_dir = Path(args.journal_dir)
|
|
journal_path = journal_dir / "improvement_journal.jsonl"
|
|
dupes = check_duplicate(journal_path, args.experiment)
|
|
if dupes:
|
|
print(f"FOUND {len(dupes)} existing entries for '{args.experiment}':")
|
|
for entry in dupes:
|
|
print(f" {entry.entry_id} ({entry.timestamp[:10]}) — SQS={entry.sqs_score}, verdict={entry.verdict}")
|
|
else:
|
|
print(f"No existing entries for '{args.experiment}'.")
|
|
|
|
|
|
def cmd_attach_wfv(args: argparse.Namespace) -> None:
|
|
"""Attach a walk-forward summary to an existing journal entry."""
|
|
journal_dir = Path(args.journal_dir)
|
|
journal_path = journal_dir / "improvement_journal.jsonl"
|
|
registry_path = journal_dir / "experiment_registry.json"
|
|
leaderboard_path = journal_dir / "LEADERBOARD.md"
|
|
summary_path = Path(args.summary)
|
|
|
|
if not journal_path.exists():
|
|
print("No journal found. Run 'record' first.")
|
|
sys.exit(1)
|
|
if not summary_path.exists():
|
|
print(f"ERROR: walk-forward summary not found: {summary_path}")
|
|
sys.exit(1)
|
|
|
|
summary = WalkForwardSummary.model_validate_json(summary_path.read_text())
|
|
with journal_lock(journal_path):
|
|
updated = attach_walk_forward_summary(journal_path, args.target, summary)
|
|
_sync_and_rebuild(journal_path, registry_path, leaderboard_path)
|
|
|
|
print(
|
|
f"Attached WFV to {updated.entry_id}: {updated.experiment_name} "
|
|
f"(WFQS={_fmt(updated.wfqs_score, '.1f')}, DEP={_fmt(updated.deployment_score, '.1f')}, folds={summary.fold_count})"
|
|
)
|
|
print(f"Leaderboard updated: {leaderboard_path}")
|
|
|
|
|
|
def cmd_attach_robustness(args: argparse.Namespace) -> None:
|
|
"""Attach a robustness matrix summary to an existing journal entry."""
|
|
journal_dir = Path(args.journal_dir)
|
|
journal_path = journal_dir / "improvement_journal.jsonl"
|
|
registry_path = journal_dir / "experiment_registry.json"
|
|
leaderboard_path = journal_dir / "LEADERBOARD.md"
|
|
summary_path = Path(args.summary)
|
|
|
|
if not journal_path.exists():
|
|
print("No journal found. Run 'record' first.")
|
|
sys.exit(1)
|
|
if not summary_path.exists():
|
|
print(f"ERROR: robustness summary not found: {summary_path}")
|
|
sys.exit(1)
|
|
|
|
summary = RobustnessMatrixSummary.model_validate_json(summary_path.read_text())
|
|
with journal_lock(journal_path):
|
|
updated = attach_robustness_summary(journal_path, args.target, summary)
|
|
_sync_and_rebuild(journal_path, registry_path, leaderboard_path)
|
|
|
|
gate_factor, _ = compute_robustness_gate(summary)
|
|
print(
|
|
f"Attached robustness matrix to {updated.entry_id}: {updated.experiment_name} "
|
|
f"(windows={summary.overall_window_count}, gate={gate_factor:.2f})"
|
|
)
|
|
print(f"Leaderboard updated: {leaderboard_path}")
|
|
|
|
|
|
def cmd_attach_oot_robustness(args: argparse.Namespace) -> None:
|
|
"""Attach out-of-time robustness matrix summary to an existing journal entry."""
|
|
journal_dir = Path(args.journal_dir)
|
|
journal_path = journal_dir / "improvement_journal.jsonl"
|
|
registry_path = journal_dir / "experiment_registry.json"
|
|
leaderboard_path = journal_dir / "LEADERBOARD.md"
|
|
summary_path = Path(args.summary)
|
|
|
|
if not journal_path.exists():
|
|
print("No journal found. Run 'record' first.")
|
|
sys.exit(1)
|
|
if not summary_path.exists():
|
|
print(f"ERROR: robustness summary not found: {summary_path}")
|
|
sys.exit(1)
|
|
|
|
summary = RobustnessMatrixSummary.model_validate_json(summary_path.read_text())
|
|
with journal_lock(journal_path):
|
|
updated = attach_out_of_time_robustness_summary(journal_path, args.target, summary)
|
|
_sync_and_rebuild(journal_path, registry_path, leaderboard_path)
|
|
|
|
gate_factor, _ = compute_oot_robustness_gate(summary)
|
|
print(
|
|
f"Attached out-of-time robustness to {updated.entry_id}: {updated.experiment_name} "
|
|
f"(windows={summary.overall_window_count}, gate={gate_factor:.2f})"
|
|
)
|
|
print(f"Leaderboard updated: {leaderboard_path}")
|
|
|
|
|
|
def cmd_attach_reset_common_window(args: argparse.Namespace) -> None:
|
|
"""Attach reset common-window summary to an existing journal entry."""
|
|
journal_dir = Path(args.journal_dir)
|
|
journal_path = journal_dir / "improvement_journal.jsonl"
|
|
registry_path = journal_dir / "experiment_registry.json"
|
|
leaderboard_path = journal_dir / "LEADERBOARD.md"
|
|
summary_path = Path(args.summary)
|
|
|
|
if not journal_path.exists():
|
|
print("No journal found. Run 'record' first.")
|
|
sys.exit(1)
|
|
if not summary_path.exists():
|
|
print(f"ERROR: reset common-window summary not found: {summary_path}")
|
|
sys.exit(1)
|
|
|
|
summary = ResetCommonWindowSummary.model_validate_json(summary_path.read_text())
|
|
with journal_lock(journal_path):
|
|
updated = attach_reset_common_window_summary(journal_path, args.target, summary)
|
|
_sync_and_rebuild(journal_path, registry_path, leaderboard_path)
|
|
|
|
print(
|
|
f"Attached reset common-window to {updated.entry_id}: {updated.experiment_name} "
|
|
f"(RCW={_fmt(updated.reset_common_window_score, '.1f')}, segments={len(summary.segment_summaries)})"
|
|
)
|
|
print(f"Leaderboard updated: {leaderboard_path}")
|
|
|
|
|
|
def _run_scenario_tests_for_missing(
|
|
journal_path: Path,
|
|
selector,
|
|
) -> int:
|
|
"""Run scenario-test for entries that match selector but lack scenario_robustness_score.
|
|
|
|
Returns the number of entries updated.
|
|
"""
|
|
from libs.backtest.scenarios.robustness import run_scenario_test
|
|
from libs.backtest.scenarios.scenarios import SCENARIO_GROUPS
|
|
from libs.backtest.tracker import attach_scenario_robustness, load_journal
|
|
|
|
entries = load_journal(journal_path)
|
|
needs_scenario = [
|
|
e for e in entries
|
|
if selector(e)
|
|
and e.scenario_robustness_score is None
|
|
and e.walk_forward_summary is not None # skip entries without WFV (SQS would be pending anyway)
|
|
]
|
|
|
|
if not needs_scenario:
|
|
return 0
|
|
|
|
print(f"\n[scenario-test] {len(needs_scenario)} entries missing RRS — running now...")
|
|
updated = 0
|
|
for entry in needs_scenario:
|
|
print(f" Running {entry.experiment_name}...", end="", flush=True)
|
|
try:
|
|
report = run_scenario_test(
|
|
experiment_name=entry.experiment_name,
|
|
scenario_names=SCENARIO_GROUPS["all"],
|
|
initial_equity=10_000.0,
|
|
)
|
|
breakdown = {
|
|
"signal_integrity": report.signal_integrity,
|
|
"breadth": report.breadth,
|
|
"drawdown_resilience": report.drawdown_resilience,
|
|
"regime_transition": report.regime_transition,
|
|
"stability": report.stability,
|
|
}
|
|
attach_scenario_robustness(journal_path, entry.experiment_name, report.rrs, breakdown)
|
|
print(f" RRS={report.rrs:.0f} verdict={report.verdict}")
|
|
updated += 1
|
|
except Exception as exc:
|
|
print(f" ERROR: {exc}")
|
|
|
|
return updated
|
|
|
|
|
|
def cmd_rescore_public(args: argparse.Namespace) -> None:
|
|
"""Recompute stored public scores for matching journal entries."""
|
|
journal_dir = Path(args.journal_dir)
|
|
journal_path = journal_dir / "improvement_journal.jsonl"
|
|
registry_path = journal_dir / "experiment_registry.json"
|
|
leaderboard_path = journal_dir / "LEADERBOARD.md"
|
|
|
|
prefix = args.prefix or ""
|
|
|
|
def _matches(entry: JournalEntry) -> bool:
|
|
if prefix and not entry.experiment_name.startswith(prefix):
|
|
return False
|
|
if args.min_version is None and args.max_version is None:
|
|
return True
|
|
if not prefix:
|
|
return False
|
|
suffix = entry.experiment_name[len(prefix):]
|
|
try:
|
|
version = float(suffix)
|
|
except ValueError:
|
|
return False
|
|
if args.min_version is not None and version < args.min_version:
|
|
return False
|
|
if args.max_version is not None and version > args.max_version:
|
|
return False
|
|
return True
|
|
|
|
with journal_lock(journal_path):
|
|
_run_scenario_tests_for_missing(journal_path, _matches)
|
|
updated_count = refresh_public_scores(journal_path, selector=_matches)
|
|
_sync_and_rebuild(journal_path, registry_path, leaderboard_path)
|
|
|
|
print(f"Rescored entries: {updated_count}")
|
|
print(f"Leaderboard updated: {leaderboard_path}")
|
|
|
|
|
|
def cmd_compute_sqs(args: argparse.Namespace) -> None:
|
|
"""Compute and display SQS v9 breakdown for a single journal entry."""
|
|
from libs.backtest.tracker import (
|
|
_resolve_journal_target,
|
|
compute_public_sqs_v9,
|
|
compute_rqs,
|
|
compute_wfqs_v2,
|
|
)
|
|
|
|
journal_dir = Path(args.journal_dir)
|
|
journal_path = journal_dir / "improvement_journal.jsonl"
|
|
registry_path = journal_dir / "experiment_registry.json"
|
|
leaderboard_path = journal_dir / "LEADERBOARD.md"
|
|
|
|
entries = load_journal(journal_path)
|
|
try:
|
|
entry = _resolve_journal_target(entries, args.target)
|
|
except ValueError as exc:
|
|
_console.print(f"[red]Error: {exc}[/red]")
|
|
sys.exit(1)
|
|
|
|
train_result = _hydrate_split_result(entry.results.get("train"))
|
|
valid_result = _hydrate_split_result(entry.results.get("valid"))
|
|
test_result = _hydrate_split_result(entry.results.get("test"))
|
|
|
|
rqs_score, rqs_breakdown = compute_rqs(train_result, valid_result, test_result)
|
|
wfqs_v2_score, wfqs_v2_breakdown = compute_wfqs_v2(entry.walk_forward_summary)
|
|
|
|
sqs_score, sqs_breakdown, source = compute_public_sqs_v9(
|
|
train_result, valid_result, test_result,
|
|
walk_forward_summary=entry.walk_forward_summary,
|
|
robustness_matrix_summary=entry.robustness_matrix_summary,
|
|
out_of_time_robustness_summary=entry.out_of_time_robustness_summary,
|
|
common_window_summary=entry.common_window_summary,
|
|
reset_common_window_summary=entry.reset_common_window_summary,
|
|
scenario_robustness_score=entry.scenario_robustness_score,
|
|
rqs_score=rqs_score,
|
|
wfqs_v2_score=wfqs_v2_score,
|
|
)
|
|
|
|
def _score_bar(score: float | None) -> str:
|
|
if score is None:
|
|
return "[dim]N/A[/dim]"
|
|
if score >= 70:
|
|
color = "green"
|
|
elif score >= 40:
|
|
color = "yellow"
|
|
else:
|
|
color = "red"
|
|
return f"[{color}]{score:.1f}[/{color}]"
|
|
|
|
regime_score = sqs_breakdown.get("regime_score")
|
|
core_score = sqs_breakdown.get("core_score")
|
|
deploy_gate = sqs_breakdown.get("deployment_gate_factor")
|
|
rb_gate = sqs_breakdown.get("gate_factor")
|
|
activity = sqs_breakdown.get("activity_factor")
|
|
|
|
regime_src = "scenario_rrs" if entry.scenario_robustness_score is not None else (
|
|
"oot_quality" if entry.out_of_time_robustness_summary is not None else "neutral_50"
|
|
)
|
|
|
|
lines = [
|
|
f"[bold]SQS v9 Breakdown[/bold]",
|
|
f"Experiment: [cyan]{entry.experiment_name}[/cyan]",
|
|
f"Entry ID: [dim]{entry.entry_id}[/dim]",
|
|
"",
|
|
f" RQS (35%): {_score_bar(rqs_score)}",
|
|
f" WFQS_v2 (40%): {_score_bar(wfqs_v2_score)}",
|
|
f" Regime (25%): {_score_bar(regime_score)} [dim]← {regime_src}[/dim]",
|
|
f" Core score: {_score_bar(core_score)}",
|
|
"",
|
|
f" Deploy gate: [dim]{deploy_gate or '?'}[/dim]",
|
|
f" Robustness gate: [dim]{rb_gate or '?'}[/dim]",
|
|
f" Activity factor: [dim]{activity or '?'}[/dim]",
|
|
"",
|
|
]
|
|
|
|
if sqs_score is not None:
|
|
# Show whether CW was blended
|
|
if "v9_3pillar+reset_cw" in (source or "") or "v9_3pillar+cw" in (source or ""):
|
|
cw_score = sqs_breakdown.get("cw_score") or sqs_breakdown.get("reset_cw_score")
|
|
lines.append(f" [bold]SQS v9: {_score_bar(sqs_score)}[/bold] [dim]{source}[/dim]")
|
|
else:
|
|
lines.append(f" [bold]SQS v9: {_score_bar(sqs_score)}[/bold] [dim]{source}[/dim]")
|
|
else:
|
|
pending_key = next((k for k in sqs_breakdown if k.startswith("requires_")), None)
|
|
reason = pending_key.replace("requires_", "").replace("_", " ") if pending_key else "missing data"
|
|
lines.append(f" [bold]SQS v9: [yellow]PENDING[/yellow][/bold] [dim]waiting for: {reason}[/dim]")
|
|
|
|
_console.print()
|
|
_console.print(Panel("\n".join(lines), box=box.ROUNDED, width=70))
|
|
|
|
# Optional: show stored vs live comparison
|
|
if entry.sqs_score is not None and sqs_score is not None:
|
|
delta = sqs_score - entry.sqs_score
|
|
sign = "+" if delta >= 0 else ""
|
|
_console.print(f" [dim]Stored SQS: {entry.sqs_score:.1f} → Live: {sqs_score:.1f} (delta {sign}{delta:.1f})[/dim]")
|
|
|
|
if args.save:
|
|
from libs.backtest.tracker import refresh_public_scores
|
|
target_name = entry.experiment_name
|
|
|
|
def _only_this(e: JournalEntry) -> bool:
|
|
return e.experiment_name == target_name
|
|
|
|
with journal_lock(journal_path):
|
|
refresh_public_scores(journal_path, selector=_only_this)
|
|
_sync_and_rebuild(journal_path, registry_path, leaderboard_path)
|
|
_console.print(f" [green]Saved and leaderboard rebuilt.[/green]")
|
|
_console.print()
|
|
|
|
|
|
def cmd_attach_overfit_check(args: argparse.Namespace) -> None:
|
|
"""Attach overfit-check score to an existing journal entry."""
|
|
journal_dir = Path(args.journal_dir)
|
|
journal_path = journal_dir / "improvement_journal.jsonl"
|
|
registry_path = journal_dir / "experiment_registry.json"
|
|
leaderboard_path = journal_dir / "LEADERBOARD.md"
|
|
|
|
if not journal_path.exists():
|
|
print("No journal found. Run 'record' first.")
|
|
sys.exit(1)
|
|
|
|
if args.report:
|
|
import json as _json
|
|
report_path = Path(args.report)
|
|
if not report_path.exists():
|
|
print(f"ERROR: report file not found: {report_path}")
|
|
sys.exit(1)
|
|
data = _json.loads(report_path.read_text())
|
|
score = float(data.get("overall_score", data.get("score", 0.0)))
|
|
breakdown: dict[str, float] = {
|
|
k: float(v) for k, v in data.items()
|
|
if isinstance(v, (int, float)) and k != "overall_score"
|
|
}
|
|
elif args.run:
|
|
from libs.backtest.overfit import run_overfit_check
|
|
from libs.backtest.tracker import load_journal as _lj
|
|
entries = _lj(journal_path)
|
|
try:
|
|
from libs.backtest.tracker import _resolve_journal_target
|
|
selected = _resolve_journal_target(entries, args.target)
|
|
except ValueError as exc:
|
|
print(f"ERROR: {exc}")
|
|
sys.exit(1)
|
|
print(f"Running overfit-check for {selected.experiment_name} (this may take 7-30 min)...")
|
|
result = run_overfit_check(selected.experiment_name, quick=args.quick)
|
|
score = result.overall_score
|
|
breakdown = result.breakdown
|
|
else:
|
|
print("ERROR: specify --report PATH or --run")
|
|
sys.exit(1)
|
|
|
|
from libs.backtest.tracker import attach_overfit_check
|
|
with journal_lock(journal_path):
|
|
updated = attach_overfit_check(journal_path, args.target, score, breakdown)
|
|
_sync_and_rebuild(journal_path, registry_path, leaderboard_path)
|
|
|
|
print(
|
|
f"Attached overfit-check to {updated.entry_id}: {updated.experiment_name} "
|
|
f"(score={score:.1f})"
|
|
)
|
|
print(f"Leaderboard updated: {leaderboard_path}")
|
|
|
|
|
|
def _print_help() -> None:
|
|
_console.print()
|
|
_console.print(Panel(
|
|
"[bold cyan]fithia2[/] — ACE-F Strategy Research & Paper Trading\n"
|
|
"[dim]백테스트 실험 추적 + Alpaca 페이퍼 트레이딩[/]",
|
|
border_style="cyan",
|
|
padding=(0, 2),
|
|
))
|
|
|
|
def _section(title: str) -> Table:
|
|
t = Table(box=box.SIMPLE, show_header=False, padding=(0, 2), show_edge=False)
|
|
t.add_column("Command", style="bold green", no_wrap=True, min_width=32)
|
|
t.add_column("Description")
|
|
t.add_column("Key Options", style="dim")
|
|
_console.print(f" [bold yellow]{title}[/]")
|
|
return t
|
|
|
|
# ── 저널 & 리더보드 ─────────────────────────────────────────
|
|
t = _section("저널 & 리더보드")
|
|
t.add_row(
|
|
"leaderboard [dim]lb[/]",
|
|
"public SQS 순위표 출력 및 LEADERBOARD.md 재생성",
|
|
"-n N --sort sqs|rqs|wfqs|deployment|promotion --include-retired",
|
|
)
|
|
t.add_row(
|
|
"record [dim]rec[/]",
|
|
"실험 결과를 저널에 기록",
|
|
"-e NAME -H TEXT -v better|worse|neutral --walk-forward-summary PATH",
|
|
)
|
|
t.add_row(
|
|
"show [dim]s[/]",
|
|
"특정 저널 항목 상세 조회",
|
|
"ENTRY_ID --diagnostics",
|
|
)
|
|
t.add_row(
|
|
"check-duplicate [dim]dup[/]",
|
|
"실험이 이미 저널에 기록됐는지 확인",
|
|
"-e NAME",
|
|
)
|
|
t.add_row(
|
|
"compute-sqs [dim]cs[/]",
|
|
"단일 항목의 SQS v9 breakdown 계산·출력",
|
|
"TARGET --save",
|
|
)
|
|
t.add_row(
|
|
"rescore-public [dim]rsp[/]",
|
|
"저장된 public 점수 일괄 재계산",
|
|
"--prefix NAME --min-version N --max-version N",
|
|
)
|
|
_console.print(t)
|
|
|
|
# ── 첨부 (attach) ───────────────────────────────────────────
|
|
t = _section("첨부 (attach)")
|
|
t.add_row(
|
|
"attach-wfv [dim]awf[/]",
|
|
"walk-forward 요약을 저널 항목에 첨부",
|
|
"TARGET --summary PATH",
|
|
)
|
|
t.add_row(
|
|
"attach-robustness [dim]arb[/]",
|
|
"robustness matrix 요약 첨부",
|
|
"TARGET --summary PATH",
|
|
)
|
|
t.add_row(
|
|
"attach-oot-robustness [dim]aoot[/]",
|
|
"out-of-time robustness 요약 첨부",
|
|
"TARGET --summary PATH",
|
|
)
|
|
t.add_row(
|
|
"attach-reset-common-window [dim]arcw[/]",
|
|
"reset common-window 요약 첨부",
|
|
"TARGET --summary PATH",
|
|
)
|
|
t.add_row(
|
|
"attach-overfit-check [dim]aoc[/]",
|
|
"overfit-check 점수 첨부 (사전 생성 리포트 또는 직접 실행)",
|
|
"TARGET --report PATH --run --quick",
|
|
)
|
|
_console.print(t)
|
|
|
|
# ── 분석 & 검증 ─────────────────────────────────────────────
|
|
t = _section("분석 & 검증")
|
|
t.add_row(
|
|
"overfit-check",
|
|
"과적합 분석 (DSR·PBO·파라미터 민감도·MC·WF Permutation)",
|
|
"--config NAME --quick --skip TESTS --output-json PATH",
|
|
)
|
|
t.add_row(
|
|
"scenario-test",
|
|
"합성 시나리오 백테스트 (Regime Robustness Score)",
|
|
"--config NAME --scenario NAME --group NAME --quick --save",
|
|
)
|
|
t.add_row(
|
|
"intraday-overfit-check",
|
|
"ORB 과적합 분석 (IS/OOS·WFV·파라미터 plateau·permutation)",
|
|
"--config NAME --split-date DATE --quick --permutations N",
|
|
)
|
|
t.add_row(
|
|
"intraday-scenario-test",
|
|
"ORB 시나리오 테스트 (연도별 regime 슬라이스 + RVOL/ranking 검증)",
|
|
"--config NAME --scenario NAME --group NAME --quick --save",
|
|
)
|
|
t.add_row(
|
|
"intraday-orb-lab",
|
|
"ORB v2 연구 파이프라인 (coarse → rerank → test → robustness → rank)",
|
|
"--config NAME --quick --beam-width N",
|
|
)
|
|
_console.print(t)
|
|
|
|
# ── 트레이딩 & 파이프라인 ────────────────────────────────────
|
|
t = _section("트레이딩 & 파이프라인")
|
|
t.add_row(
|
|
"paper",
|
|
"Alpaca 페이퍼 트레이딩 [dim](fithia2 paper 로 상세 확인)[/]",
|
|
"start run run-open run-close positions status trades equity auto ...",
|
|
)
|
|
t.add_row(
|
|
"pipeline",
|
|
"데이터 파이프라인 실행 [dim](fithia2 pipeline 로 상세 확인)[/]",
|
|
"run --step poller|fetcher|parser|features|labels",
|
|
)
|
|
t.add_row(
|
|
"refresh [dim][snapshot_id][/]",
|
|
"Parquet 스냅샷 새로고침 (미지정 시 configs에서 자동 감지)",
|
|
"[snapshot_id]",
|
|
)
|
|
t.add_row(
|
|
"web",
|
|
"웹 GUI 서버 실행",
|
|
"",
|
|
)
|
|
_console.print(t)
|
|
|
|
# ── 실험 관리 ────────────────────────────────────────────────
|
|
t = _section("실험 관리")
|
|
t.add_row(
|
|
"exp",
|
|
"실험 메타데이터 관리 [dim](fithia2 exp 로 상세 확인)[/]",
|
|
"create search tree info diff promote retire validate migrate rebuild-index",
|
|
)
|
|
_console.print(t)
|
|
|
|
_console.print(
|
|
" [dim]공통 옵션:[/] [bold]--journal-dir[/] [dim](기본: journal/)[/]"
|
|
" [bold]--runs-dir[/] [dim](기본: runs/)[/]\n"
|
|
)
|
|
_console.print(" [bold]예시[/]")
|
|
_console.print(" [green]fithia2 leaderboard[/]")
|
|
_console.print(" [green]fithia2 record -e pead_v2 -H '...' -v better[/]")
|
|
_console.print(" [green]fithia2 compute-sqs return_max_long_v7.119_composed_gld[/]")
|
|
_console.print(" [green]fithia2 overfit-check --config return_max_long_v7.119 --quick[/]")
|
|
_console.print(" [green]fithia2 paper status[/]")
|
|
_console.print(" [green]fithia2 refresh[/]")
|
|
_console.print()
|
|
|
|
|
|
def main() -> None:
|
|
# Delegate `fithia2 exp ...` to the experiment management CLI
|
|
if len(sys.argv) >= 2 and sys.argv[1] == "exp":
|
|
sys.argv = [sys.argv[0]] + sys.argv[2:]
|
|
from apps.experiment.cli import main as exp_main
|
|
exp_main()
|
|
return
|
|
|
|
# Delegate `fithia2 overfit-check ...` to the overfitting analysis CLI
|
|
if len(sys.argv) >= 2 and sys.argv[1] == "overfit-check":
|
|
sys.argv = [sys.argv[0]] + sys.argv[2:]
|
|
from apps.overfit.cli import main as overfit_main
|
|
overfit_main()
|
|
return
|
|
|
|
# Delegate `fithia2 scenario-test ...` to the synthetic scenario test CLI
|
|
if len(sys.argv) >= 2 and sys.argv[1] == "scenario-test":
|
|
sys.argv = [sys.argv[0]] + sys.argv[2:]
|
|
from apps.scenario.cli import main as scenario_main
|
|
scenario_main()
|
|
return
|
|
|
|
# Delegate `fithia2 intraday-overfit-check ...` to the ORB overfitting analysis CLI
|
|
if len(sys.argv) >= 2 and sys.argv[1] == "intraday-overfit-check":
|
|
sys.argv = [sys.argv[0]] + sys.argv[2:]
|
|
from apps.intraday_bt.overfit_check import main as intraday_overfit_main
|
|
intraday_overfit_main()
|
|
return
|
|
|
|
# Delegate `fithia2 intraday-scenario-test ...` to the ORB scenario test CLI
|
|
if len(sys.argv) >= 2 and sys.argv[1] == "intraday-scenario-test":
|
|
sys.argv = [sys.argv[0]] + sys.argv[2:]
|
|
from apps.intraday_bt.scenario_test import main as intraday_scenario_main
|
|
intraday_scenario_main()
|
|
return
|
|
|
|
# Delegate `fithia2 intraday-orb-lab ...` to the ORB research lab CLI
|
|
if len(sys.argv) >= 2 and sys.argv[1] == "intraday-orb-lab":
|
|
sys.argv = [sys.argv[0]] + sys.argv[2:]
|
|
from apps.intraday_bt.lab import main as intraday_orb_lab_main
|
|
intraday_orb_lab_main()
|
|
return
|
|
|
|
# Delegate `fithia2 paper ...` to the paper trader CLI
|
|
if len(sys.argv) >= 2 and sys.argv[1] == "paper":
|
|
sys.argv = [sys.argv[0]] + sys.argv[2:]
|
|
from apps.paper_trader.cli import main as paper_main
|
|
paper_main()
|
|
return
|
|
|
|
# fithia2 refresh [snapshot_id] — refresh Parquet snapshot
|
|
if len(sys.argv) >= 2 and sys.argv[1] == "refresh":
|
|
import asyncio
|
|
from apps.paper_trader.backtest_sim import _refresh_snapshot
|
|
from libs.backtest.snapshots import resolve_snapshot
|
|
from rich.console import Console
|
|
console = Console()
|
|
|
|
snapshot_id = sys.argv[2] if len(sys.argv) >= 3 else None
|
|
if not snapshot_id:
|
|
# Auto-detect: use the most common snapshot_id from configs
|
|
from pathlib import Path
|
|
import json
|
|
configs_dir = Path("configs/experiments")
|
|
if configs_dir.exists():
|
|
ids: dict[str, int] = {}
|
|
for f in configs_dir.glob("*.json"):
|
|
try:
|
|
data = json.loads(f.read_text())
|
|
sid = data.get("dataset_snapshot_id", "")
|
|
if sid:
|
|
ids[sid] = ids.get(sid, 0) + 1
|
|
except Exception:
|
|
pass
|
|
if ids:
|
|
snapshot_id = max(ids, key=ids.get)
|
|
console.print(f"[dim]Auto-detected snapshot: {snapshot_id}[/]")
|
|
if not snapshot_id:
|
|
console.print("[red]Usage: fithia2 refresh <snapshot_id>[/]")
|
|
sys.exit(1)
|
|
|
|
universe_profile = None
|
|
if "midlarge" in snapshot_id:
|
|
universe_profile = "midlarge-liquid-long-v1"
|
|
elif "midwide" in snapshot_id:
|
|
universe_profile = "midwide-liquid-long-v1"
|
|
elif "smallcap" in snapshot_id:
|
|
universe_profile = "smallcap-liquid-long-v1"
|
|
|
|
try:
|
|
resolution = resolve_snapshot(snapshot_id)
|
|
if resolution.requested_snapshot_id != resolution.canonical_snapshot_id:
|
|
console.print(
|
|
f"[bold]Refreshing snapshot:[/] {snapshot_id} "
|
|
f"[dim](resolved → {resolution.canonical_snapshot_id})[/]"
|
|
)
|
|
else:
|
|
console.print(f"[bold]Refreshing snapshot:[/] {snapshot_id}")
|
|
except Exception:
|
|
console.print(f"[bold]Refreshing snapshot:[/] {snapshot_id}")
|
|
try:
|
|
asyncio.run(_refresh_snapshot(snapshot_id, universe_profile, console=console, manual=True))
|
|
console.print("[bold green]Done.[/]")
|
|
except Exception as exc:
|
|
console.print(f"[bold red]Refresh failed: {exc}[/]")
|
|
sys.exit(1)
|
|
return
|
|
|
|
# Delegate `fithia2 pipeline ...` to the pipeline CLI
|
|
if len(sys.argv) >= 2 and sys.argv[1] == "pipeline":
|
|
sys.argv = [sys.argv[0]] + sys.argv[2:]
|
|
from apps.pipeline.cli import main as pipeline_main
|
|
pipeline_main()
|
|
return
|
|
|
|
# Delegate `fithia2 web ...` to the web GUI server
|
|
if len(sys.argv) >= 2 and sys.argv[1] == "web":
|
|
from apps.web.main import run_server
|
|
run_server()
|
|
return
|
|
|
|
if len(sys.argv) == 1:
|
|
_print_help()
|
|
sys.exit(0)
|
|
|
|
parser = argparse.ArgumentParser(description="Strategy Improvement Tracker", add_help=True)
|
|
sub = parser.add_subparsers(dest="command", required=True)
|
|
|
|
journal_kwargs = {
|
|
"default": _DEFAULT_JOURNAL_DIR,
|
|
"help": f"Path to journal/ directory (default: {_DEFAULT_JOURNAL_DIR})",
|
|
}
|
|
|
|
for name in ("record", "rec"):
|
|
subparser = sub.add_parser(name, help="Record an experiment to the journal")
|
|
subparser.add_argument("--journal-dir", **journal_kwargs)
|
|
subparser.add_argument("--runs-dir", default="runs", help="Path to runs output directory (default: runs)")
|
|
subparser.add_argument("--experiment", "-e", required=True, help="Experiment name (matches manifest)")
|
|
subparser.add_argument("--hypothesis", "-H", help="What you expected this change to do")
|
|
subparser.add_argument("--baseline", "-b", help="Baseline experiment name for comparison")
|
|
subparser.add_argument("--verdict", "-v", choices=["better", "worse", "neutral", "unknown"], default="unknown")
|
|
subparser.add_argument("--reasoning", "-r", help="Why this verdict")
|
|
subparser.add_argument("--next", "-n", help="Next experiment direction")
|
|
subparser.add_argument("--walk-forward-summary", help="Optional path to walk_forward_summary.json")
|
|
subparser.add_argument("--robustness-summary", help="Optional path to robustness_matrix_summary.json")
|
|
subparser.add_argument(
|
|
"--out-of-time-robustness-summary",
|
|
help="Optional path to out-of-time robustness_matrix_summary.json",
|
|
)
|
|
subparser.add_argument(
|
|
"--reset-common-window-summary",
|
|
help="Optional path to reset_common_window_summary.json",
|
|
)
|
|
subparser.add_argument("--force", "-f", action="store_true", help="Allow duplicate experiment names")
|
|
|
|
for name in ("leaderboard", "lb"):
|
|
subparser = sub.add_parser(name, help="Show/regenerate the leaderboard")
|
|
subparser.add_argument("--journal-dir", **journal_kwargs)
|
|
subparser.add_argument("--top", "-n", type=int, default=10, help="Show top N entries (default: 10)")
|
|
subparser.add_argument(
|
|
"--sort",
|
|
choices=["sqs", "rqs", "wfqs", "deployment", "dep", "promotion", "unified", "sqs2"],
|
|
default="sqs",
|
|
help="Sort by public SQS (default) or internal diagnostics",
|
|
)
|
|
subparser.add_argument(
|
|
"--include-retired",
|
|
action="store_true",
|
|
help="Include retired pre-IMP-0606 research and retired legacy families",
|
|
)
|
|
|
|
for name in ("show", "s"):
|
|
subparser = sub.add_parser(name, help="Show details of a journal entry")
|
|
subparser.add_argument("--journal-dir", **journal_kwargs)
|
|
subparser.add_argument("entry_id", help="Entry ID (IMP-0001) or experiment name")
|
|
subparser.add_argument("--diagnostics", action="store_true", help="Show internal RQS/WFQS/DEP diagnostics")
|
|
|
|
for name in ("check-duplicate", "dup"):
|
|
subparser = sub.add_parser(name, help="Check if experiment already recorded")
|
|
subparser.add_argument("--journal-dir", **journal_kwargs)
|
|
subparser.add_argument("--experiment", "-e", required=True, help="Experiment name to check")
|
|
|
|
for name in ("attach-wfv", "awf"):
|
|
subparser = sub.add_parser(name, help="Attach walk-forward summary to an existing journal entry")
|
|
subparser.add_argument("--journal-dir", **journal_kwargs)
|
|
subparser.add_argument("target", help="Entry ID or experiment name to update")
|
|
subparser.add_argument("--summary", required=True, help="Path to walk_forward_summary.json")
|
|
|
|
for name in ("attach-robustness", "arb"):
|
|
subparser = sub.add_parser(name, help="Attach robustness matrix summary to an existing journal entry")
|
|
subparser.add_argument("--journal-dir", **journal_kwargs)
|
|
subparser.add_argument("target", help="Entry ID or experiment name to update")
|
|
subparser.add_argument("--summary", required=True, help="Path to robustness_matrix_summary.json")
|
|
|
|
for name in ("attach-oot-robustness", "aoot"):
|
|
subparser = sub.add_parser(name, help="Attach out-of-time robustness summary to an existing journal entry")
|
|
subparser.add_argument("--journal-dir", **journal_kwargs)
|
|
subparser.add_argument("target", help="Entry ID or experiment name to update")
|
|
subparser.add_argument("--summary", required=True, help="Path to robustness_matrix_summary.json")
|
|
|
|
for name in ("attach-reset-common-window", "arcw"):
|
|
subparser = sub.add_parser(name, help="Attach reset common-window summary to an existing journal entry")
|
|
subparser.add_argument("--journal-dir", **journal_kwargs)
|
|
subparser.add_argument("target", help="Entry ID or experiment name to update")
|
|
subparser.add_argument("--summary", required=True, help="Path to reset_common_window_summary.json")
|
|
|
|
for name in ("rescore-public", "rsp"):
|
|
subparser = sub.add_parser(name, help="Recompute stored public scores for journal entries")
|
|
subparser.add_argument("--journal-dir", **journal_kwargs)
|
|
subparser.add_argument("--prefix", help="Only rescore experiment names with this prefix")
|
|
subparser.add_argument("--min-version", type=float, help="Minimum numeric suffix after --prefix")
|
|
subparser.add_argument("--max-version", type=float, help="Maximum numeric suffix after --prefix")
|
|
|
|
for name in ("compute-sqs", "cs"):
|
|
subparser = sub.add_parser(name, help="Compute and display SQS v9 breakdown for a single entry")
|
|
subparser.add_argument("--journal-dir", **journal_kwargs)
|
|
subparser.add_argument("target", help="Entry ID, experiment name, or partial name")
|
|
subparser.add_argument("--save", action="store_true", help="Save recomputed score back to journal and rebuild leaderboard")
|
|
|
|
for name in ("attach-overfit-check", "aoc"):
|
|
subparser = sub.add_parser(name, help="Attach overfit-check score to an existing journal entry")
|
|
subparser.add_argument("--journal-dir", **journal_kwargs)
|
|
subparser.add_argument("target", help="Entry ID or experiment name to update")
|
|
subparser.add_argument("--report", default=None, help="Path to pre-generated overfit-check JSON report")
|
|
subparser.add_argument("--run", action="store_true", help="Run overfit-check directly (7-30 min)")
|
|
subparser.add_argument("--quick", action="store_true", help="Quick mode when using --run")
|
|
|
|
args = parser.parse_args()
|
|
dispatch = {
|
|
"record": cmd_record,
|
|
"rec": cmd_record,
|
|
"leaderboard": cmd_leaderboard,
|
|
"lb": cmd_leaderboard,
|
|
"show": cmd_show,
|
|
"s": cmd_show,
|
|
"check-duplicate": cmd_check_duplicate,
|
|
"dup": cmd_check_duplicate,
|
|
"attach-wfv": cmd_attach_wfv,
|
|
"awf": cmd_attach_wfv,
|
|
"attach-robustness": cmd_attach_robustness,
|
|
"arb": cmd_attach_robustness,
|
|
"attach-oot-robustness": cmd_attach_oot_robustness,
|
|
"aoot": cmd_attach_oot_robustness,
|
|
"attach-reset-common-window": cmd_attach_reset_common_window,
|
|
"arcw": cmd_attach_reset_common_window,
|
|
"rescore-public": cmd_rescore_public,
|
|
"rsp": cmd_rescore_public,
|
|
"compute-sqs": cmd_compute_sqs,
|
|
"cs": cmd_compute_sqs,
|
|
"attach-overfit-check": cmd_attach_overfit_check,
|
|
"aoc": cmd_attach_overfit_check,
|
|
}
|
|
dispatch[args.command](args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|