You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
967 lines
40 KiB
Python
967 lines
40 KiB
Python
"""CLI for strategy improvement tracking: record, leaderboard, check-duplicate."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from rich import box
|
|
from rich.console import Console
|
|
from rich.panel import Panel
|
|
from rich.table import Table
|
|
|
|
from libs.backtest.domain import (
|
|
ConfigDelta,
|
|
JournalEntry,
|
|
RobustnessMatrixSummary,
|
|
SplitResult,
|
|
WalkForwardSummary,
|
|
)
|
|
from libs.backtest.tracker import (
|
|
_hydrate_split_result,
|
|
append_journal_entry,
|
|
attach_out_of_time_robustness_summary,
|
|
attach_robustness_summary,
|
|
attach_walk_forward_summary,
|
|
build_split_result,
|
|
check_duplicate,
|
|
compute_deployment_score,
|
|
compute_public_sqs,
|
|
compute_public_sqs_v2,
|
|
compute_promotion_score,
|
|
compute_oot_robustness_gate,
|
|
compute_robustness_gate,
|
|
compute_rqs,
|
|
compute_sqs,
|
|
compute_sqs_v2,
|
|
compute_unified_score,
|
|
compute_wfqs,
|
|
compute_wfqs_v2,
|
|
filter_registry_entries,
|
|
get_next_entry_id,
|
|
journal_lock,
|
|
load_journal,
|
|
refresh_public_scores,
|
|
rebuild_registry,
|
|
scan_runs_for_experiment,
|
|
sync_official_manifests,
|
|
)
|
|
from libs.common.time_utils import utc_now
|
|
|
|
_console = Console(width=140)
|
|
_DEFAULT_JOURNAL_DIR = "journal"
|
|
_COL_NAME = 38
|
|
|
|
|
|
def _fmt(val: float | None, fmt: str) -> str:
|
|
return format(val, fmt) if val is not None else "-"
|
|
|
|
|
|
def _ratio(numerator: float | None, denominator: float | None) -> float | None:
|
|
if numerator is None or denominator is None or denominator == 0:
|
|
return None
|
|
return numerator / denominator
|
|
|
|
|
|
def _entry_matches_target(entry: JournalEntry, target: str) -> bool:
|
|
target_upper = target.upper()
|
|
return entry.entry_id == target_upper or entry.experiment_name == target
|
|
|
|
|
|
def _ordered_splits(results: dict[str, SplitResult]) -> list[tuple[str, SplitResult]]:
|
|
preferred = ["train", "valid", "test"]
|
|
ordered: list[tuple[str, SplitResult]] = []
|
|
seen: set[str] = set()
|
|
for name in preferred:
|
|
if name in results:
|
|
ordered.append((name, results[name]))
|
|
seen.add(name)
|
|
for name in sorted(results.keys()):
|
|
if name not in seen:
|
|
ordered.append((name, results[name]))
|
|
return ordered
|
|
|
|
|
|
def _sync_and_rebuild(
|
|
journal_path: Path,
|
|
registry_path: Path,
|
|
leaderboard_path: Path,
|
|
) -> object:
|
|
sync_official_manifests(journal_path, Path("runs"))
|
|
return rebuild_registry(journal_path, registry_path, leaderboard_path)
|
|
|
|
|
|
def _load_optional_wfv_summary(path_str: str | None) -> WalkForwardSummary | None:
|
|
if not path_str:
|
|
return None
|
|
path = Path(path_str)
|
|
if not path.exists():
|
|
print(f"ERROR: walk-forward summary not found: {path}")
|
|
sys.exit(1)
|
|
return WalkForwardSummary.model_validate_json(path.read_text())
|
|
|
|
|
|
def _load_optional_robustness_summary(path_str: str | None) -> RobustnessMatrixSummary | None:
|
|
if not path_str:
|
|
return None
|
|
path = Path(path_str)
|
|
if not path.exists():
|
|
print(f"ERROR: robustness summary not found: {path}")
|
|
sys.exit(1)
|
|
return RobustnessMatrixSummary.model_validate_json(path.read_text())
|
|
|
|
|
|
def _score_sort_key(sort_by: str, entry) -> tuple:
|
|
if sort_by == "promotion":
|
|
return (
|
|
entry.promotion_score is None,
|
|
-(entry.promotion_score or 0.0),
|
|
-(entry.sqs_score or 0.0),
|
|
)
|
|
if sort_by in {"deployment", "dep"}:
|
|
return (
|
|
entry.deployment_score is None,
|
|
-(entry.deployment_score or 0.0),
|
|
entry.wfqs_score is None,
|
|
-(entry.wfqs_score or 0.0),
|
|
-(entry.rqs_score or 0.0),
|
|
-(entry.sqs_score or 0.0),
|
|
)
|
|
if sort_by == "wfqs":
|
|
return (
|
|
entry.wfqs_score is None,
|
|
-(entry.wfqs_score or 0.0),
|
|
-(entry.deployment_score or 0.0),
|
|
-(entry.rqs_score or 0.0),
|
|
-(entry.sqs_score or 0.0),
|
|
)
|
|
if sort_by == "rqs":
|
|
return (
|
|
entry.rqs_score is None,
|
|
-(entry.rqs_score or 0.0),
|
|
-(entry.deployment_score or 0.0),
|
|
-(entry.sqs_score or 0.0),
|
|
)
|
|
if sort_by == "unified":
|
|
return (
|
|
entry.unified_score is None,
|
|
-(entry.unified_score or 0.0),
|
|
-(entry.sqs_score or 0.0),
|
|
)
|
|
if sort_by == "sqs2":
|
|
return (
|
|
entry.wfqs_v2_score is None,
|
|
-(entry.wfqs_v2_score or 0.0),
|
|
-(entry.wfqs_score or 0.0),
|
|
-(entry.sqs_score or 0.0),
|
|
)
|
|
return (-(entry.sqs_score or 0.0),)
|
|
|
|
|
|
def _validation_pending_reason(entry: JournalEntry) -> str | None:
|
|
missing: list[str] = []
|
|
if entry.walk_forward_summary is None:
|
|
missing.append("WFV")
|
|
if entry.robustness_matrix_summary is None:
|
|
missing.append("robustness")
|
|
if entry.out_of_time_robustness_summary is None:
|
|
missing.append("OOT robustness")
|
|
if not missing:
|
|
return None
|
|
return "missing " + " + ".join(missing)
|
|
|
|
|
|
def _title_for_sort(sort_by: str) -> str:
|
|
if sort_by == "promotion":
|
|
return "Promotion 기준 내림차순"
|
|
if sort_by in {"deployment", "dep"}:
|
|
return "Deployment 기준 내림차순"
|
|
if sort_by == "wfqs":
|
|
return "WFQS 기준 내림차순"
|
|
if sort_by == "rqs":
|
|
return "RQS 기준 내림차순"
|
|
if sort_by == "unified":
|
|
return "Unified 기준 내림차순"
|
|
if sort_by == "sqs2":
|
|
return "WFQS v2 기준 내림차순"
|
|
return "SQS 기준 내림차순"
|
|
|
|
|
|
def cmd_record(args: argparse.Namespace) -> None:
|
|
"""Record an experiment to the improvement journal."""
|
|
journal_dir = Path(args.journal_dir)
|
|
journal_path = journal_dir / "improvement_journal.jsonl"
|
|
registry_path = journal_dir / "experiment_registry.json"
|
|
leaderboard_path = journal_dir / "LEADERBOARD.md"
|
|
|
|
runs_dir = Path(args.runs_dir)
|
|
if not runs_dir.exists():
|
|
print(f"ERROR: runs directory not found: {runs_dir}")
|
|
sys.exit(1)
|
|
|
|
split_runs = scan_runs_for_experiment(runs_dir, args.experiment)
|
|
if not split_runs:
|
|
print(f"ERROR: no runs found for experiment '{args.experiment}' in {runs_dir}")
|
|
sys.exit(1)
|
|
|
|
results: dict[str, SplitResult] = {}
|
|
for split_name, (run_id, metrics) in split_runs.items():
|
|
results[split_name] = build_split_result(split_name, run_id, metrics)
|
|
|
|
test_metrics = None
|
|
for preferred in ["test", "valid", "train"]:
|
|
if preferred in split_runs:
|
|
_, test_metrics = split_runs[preferred]
|
|
break
|
|
|
|
sqs_score = None
|
|
sqs_breakdown: dict[str, float] = {}
|
|
sqs_v2_score = None
|
|
sqs_v2_breakdown: dict[str, float] = {}
|
|
if test_metrics is not None:
|
|
sqs_score, sqs_breakdown = compute_sqs(test_metrics)
|
|
sqs_v2_score, sqs_v2_breakdown = compute_sqs_v2(test_metrics)
|
|
|
|
promotion_score, promotion_breakdown = compute_promotion_score(
|
|
results.get("test"),
|
|
results.get("valid"),
|
|
)
|
|
unified_score, unified_breakdown = compute_unified_score(
|
|
results.get("test"),
|
|
results.get("valid"),
|
|
)
|
|
rqs_score, rqs_breakdown = compute_rqs(
|
|
results.get("train"),
|
|
results.get("valid"),
|
|
results.get("test"),
|
|
)
|
|
|
|
walk_forward_summary = _load_optional_wfv_summary(args.walk_forward_summary)
|
|
robustness_matrix_summary = _load_optional_robustness_summary(args.robustness_summary)
|
|
out_of_time_robustness_summary = _load_optional_robustness_summary(
|
|
getattr(args, "out_of_time_robustness_summary", None)
|
|
)
|
|
|
|
wfqs_score = None
|
|
wfqs_breakdown: dict[str, float] = {}
|
|
wfqs_v2_score = None
|
|
wfqs_v2_breakdown: dict[str, float] = {}
|
|
deployment_score = None
|
|
deployment_breakdown: dict[str, float] = {}
|
|
if walk_forward_summary is not None:
|
|
wfqs_score, wfqs_breakdown = compute_wfqs(walk_forward_summary)
|
|
wfqs_v2_score, wfqs_v2_breakdown = compute_wfqs_v2(walk_forward_summary)
|
|
deployment_score, deployment_breakdown = compute_deployment_score(
|
|
results.get("train"),
|
|
results.get("valid"),
|
|
results.get("test"),
|
|
walk_forward_summary,
|
|
rqs_score=rqs_score,
|
|
wfqs_score=wfqs_score,
|
|
)
|
|
|
|
public_sqs, public_breakdown, public_source = compute_public_sqs(
|
|
results.get("train"),
|
|
results.get("valid"),
|
|
results.get("test"),
|
|
walk_forward_summary=walk_forward_summary,
|
|
robustness_matrix_summary=robustness_matrix_summary,
|
|
out_of_time_robustness_summary=out_of_time_robustness_summary,
|
|
rqs_score=rqs_score,
|
|
wfqs_score=wfqs_v2_score,
|
|
)
|
|
stress_sqs, stress_breakdown, _ = compute_public_sqs_v2(
|
|
results.get("train"),
|
|
results.get("valid"),
|
|
results.get("test"),
|
|
walk_forward_summary=walk_forward_summary,
|
|
robustness_matrix_summary=robustness_matrix_summary,
|
|
out_of_time_robustness_summary=out_of_time_robustness_summary,
|
|
rqs_score=rqs_score,
|
|
wfqs_v2_score=wfqs_v2_score,
|
|
)
|
|
if public_sqs is not None:
|
|
sqs_score = public_sqs
|
|
sqs_breakdown = public_breakdown
|
|
|
|
config_delta = None
|
|
if args.baseline:
|
|
config_delta = ConfigDelta(base_experiment=args.baseline, changes={})
|
|
|
|
tags = [tag for tag in args.experiment.replace("-", "_").split("_") if tag]
|
|
|
|
with journal_lock(journal_path):
|
|
dupes = check_duplicate(journal_path, args.experiment)
|
|
if dupes and not args.force:
|
|
print(f"WARNING: experiment '{args.experiment}' already in journal ({len(dupes)} entries).")
|
|
print("Use --force to add anyway.")
|
|
sys.exit(1)
|
|
|
|
entry_id = get_next_entry_id(journal_path)
|
|
entry = JournalEntry(
|
|
entry_id=entry_id,
|
|
timestamp=utc_now().isoformat(),
|
|
experiment_name=args.experiment,
|
|
hypothesis=args.hypothesis or "",
|
|
config_delta=config_delta,
|
|
results=results,
|
|
walk_forward_summary=walk_forward_summary,
|
|
robustness_matrix_summary=robustness_matrix_summary,
|
|
out_of_time_robustness_summary=out_of_time_robustness_summary,
|
|
sqs_score=sqs_score,
|
|
sqs_breakdown=sqs_breakdown,
|
|
stress_sqs_score=stress_sqs,
|
|
stress_sqs_breakdown=stress_breakdown,
|
|
sqs_v2_score=sqs_v2_score,
|
|
sqs_v2_breakdown=sqs_v2_breakdown,
|
|
promotion_score=promotion_score,
|
|
promotion_breakdown=promotion_breakdown,
|
|
unified_score=unified_score,
|
|
unified_breakdown=unified_breakdown,
|
|
rqs_score=rqs_score,
|
|
rqs_breakdown=rqs_breakdown,
|
|
wfqs_score=wfqs_score,
|
|
wfqs_breakdown=wfqs_breakdown,
|
|
wfqs_v2_score=wfqs_v2_score,
|
|
wfqs_v2_breakdown=wfqs_v2_breakdown,
|
|
deployment_score=deployment_score,
|
|
deployment_breakdown=deployment_breakdown,
|
|
verdict=args.verdict or "unknown",
|
|
verdict_reasoning=args.reasoning or "",
|
|
next_direction=args.next or "",
|
|
tags=tags,
|
|
)
|
|
|
|
append_journal_entry(journal_path, entry)
|
|
_sync_and_rebuild(journal_path, registry_path, leaderboard_path)
|
|
|
|
if sqs_score is None:
|
|
print(f"Recorded {entry_id}: {args.experiment} (SQS=pending validation)")
|
|
else:
|
|
source_label = f", source={public_source}" if public_source else ""
|
|
stress_label = f", stress={stress_sqs:.1f}" if stress_sqs is not None else ""
|
|
print(f"Recorded {entry_id}: {args.experiment} (SQS={sqs_score}{stress_label}{source_label})")
|
|
for split_name, sr in _ordered_splits(results):
|
|
pf = f"PF={sr.profit_factor:.2f}" if sr.profit_factor is not None else "PF=-"
|
|
ret = f"Ret={sr.total_return_pct:+.2f}%" if sr.total_return_pct is not None else "Ret=-"
|
|
print(f" {split_name}: {sr.trade_count} trades, {pf}, {ret}")
|
|
print(f"Leaderboard updated: {leaderboard_path}")
|
|
|
|
|
|
def cmd_leaderboard(args: argparse.Namespace) -> None:
|
|
"""Show or regenerate the leaderboard."""
|
|
journal_dir = Path(args.journal_dir)
|
|
journal_path = journal_dir / "improvement_journal.jsonl"
|
|
registry_path = journal_dir / "experiment_registry.json"
|
|
leaderboard_path = journal_dir / "LEADERBOARD.md"
|
|
|
|
if not journal_path.exists():
|
|
journal_path.parent.mkdir(parents=True, exist_ok=True)
|
|
journal_path.touch()
|
|
|
|
with journal_lock(journal_path):
|
|
registry = _sync_and_rebuild(journal_path, registry_path, leaderboard_path)
|
|
ranked_source = filter_registry_entries(
|
|
registry.entries,
|
|
include_retired=getattr(args, "include_retired", False),
|
|
)
|
|
displayed_leaderboard_path = leaderboard_path
|
|
|
|
sort_by = getattr(args, "sort", "sqs")
|
|
ranked_entries = sorted(ranked_source, key=lambda entry: _score_sort_key(sort_by, entry))
|
|
total = len(ranked_entries)
|
|
top_n = getattr(args, "top", 10)
|
|
diagnostics = sort_by in {"deployment", "dep", "wfqs", "rqs", "promotion", "unified", "sqs2"}
|
|
title = f"[bold cyan]Top {top_n} / {total}[/] [dim]· {_title_for_sort(sort_by)} · Tr=train V=valid T=test[/]"
|
|
|
|
tbl = Table(
|
|
box=box.SIMPLE_HEAD,
|
|
show_header=True,
|
|
header_style="bold yellow",
|
|
row_styles=["", "dim"],
|
|
padding=(0, 1),
|
|
title=title,
|
|
title_justify="left",
|
|
expand=False,
|
|
)
|
|
tbl.add_column("#", justify="right", style="bold", no_wrap=True, min_width=3)
|
|
tbl.add_column("Experiment", no_wrap=True, min_width=30)
|
|
tbl.add_column("SQS", justify="right", style="bold cyan", no_wrap=True, min_width=5)
|
|
if diagnostics:
|
|
if sort_by == "promotion":
|
|
tbl.add_column("Promo", justify="right", style="bright_magenta", no_wrap=True, min_width=5)
|
|
else:
|
|
tbl.add_column("DEP", justify="right", style="bold green", no_wrap=True, min_width=5)
|
|
tbl.add_column("WFQS", justify="right", style="bright_cyan", no_wrap=True, min_width=5)
|
|
tbl.add_column("RQS", justify="right", style="bold magenta", no_wrap=True, min_width=5)
|
|
tbl.add_column("Tr.Ret%", justify="right", style="bright_blue", no_wrap=True, min_width=7)
|
|
tbl.add_column("V.Ret%", justify="right", style="green", no_wrap=True, min_width=6)
|
|
tbl.add_column("T.Ret%", justify="right", no_wrap=True, min_width=6)
|
|
tbl.add_column("T.Ann%", justify="right", no_wrap=True, min_width=7)
|
|
tbl.add_column("T.DD%", justify="right", no_wrap=True, min_width=5)
|
|
tbl.add_column("T.Gross%", justify="right", no_wrap=True, min_width=8)
|
|
tbl.add_column("T.DIM%", justify="right", no_wrap=True, min_width=6)
|
|
tbl.add_column("T.R/G", justify="right", no_wrap=True, min_width=5)
|
|
|
|
for rank, entry in enumerate(ranked_entries[:top_n], 1):
|
|
name = entry.experiment_name
|
|
if len(name) > _COL_NAME:
|
|
name = name[: _COL_NAME - 1] + "…"
|
|
row = [
|
|
str(rank),
|
|
name,
|
|
_fmt(entry.sqs_score, ".1f"),
|
|
]
|
|
if diagnostics:
|
|
if sort_by == "promotion":
|
|
row.append(_fmt(entry.promotion_score, ".1f"))
|
|
else:
|
|
row.extend([
|
|
_fmt(entry.deployment_score, ".1f"),
|
|
_fmt(entry.wfqs_score, ".1f"),
|
|
_fmt(entry.rqs_score, ".1f"),
|
|
])
|
|
row.extend([
|
|
_fmt(entry.train_total_return_pct, "+.1f"),
|
|
_fmt(entry.valid_total_return_pct, "+.1f"),
|
|
_fmt(entry.total_return_pct, "+.1f"),
|
|
_fmt(entry.annualized_return_pct, "+.1f"),
|
|
_fmt(entry.max_drawdown_pct, ".1f"),
|
|
_fmt(entry.avg_gross_exposure_pct, ".1f"),
|
|
_fmt(entry.days_in_market_pct, ".1f"),
|
|
_fmt(_ratio(entry.total_return_pct, entry.avg_gross_exposure_pct), ".2f"),
|
|
])
|
|
tbl.add_row(*row)
|
|
|
|
_console.print()
|
|
_console.print(tbl)
|
|
_console.print(f" [dim]LEADERBOARD.md → {displayed_leaderboard_path}[/]\n")
|
|
|
|
|
|
def cmd_show(args: argparse.Namespace) -> None:
|
|
"""Show details of a specific journal entry."""
|
|
journal_dir = Path(args.journal_dir)
|
|
journal_path = journal_dir / "improvement_journal.jsonl"
|
|
registry_path = journal_dir / "experiment_registry.json"
|
|
leaderboard_path = journal_dir / "LEADERBOARD.md"
|
|
with journal_lock(journal_path):
|
|
_sync_and_rebuild(journal_path, registry_path, leaderboard_path)
|
|
entries = load_journal(journal_path)
|
|
|
|
target = args.entry_id
|
|
found = [entry for entry in entries if _entry_matches_target(entry, target)]
|
|
if not found:
|
|
found = [entry for entry in entries if target.lower() in entry.experiment_name.lower()]
|
|
if not found:
|
|
print(f"No entry found for: {target}")
|
|
sys.exit(1)
|
|
|
|
for entry in found:
|
|
train_result = _hydrate_split_result(entry.results.get("train"))
|
|
valid_result = _hydrate_split_result(entry.results.get("valid"))
|
|
test_result = _hydrate_split_result(entry.results.get("test"))
|
|
fresh_rqs_score, fresh_rqs_breakdown = compute_rqs(
|
|
train_result,
|
|
valid_result,
|
|
test_result,
|
|
)
|
|
fresh_wfqs_score, fresh_wfqs_breakdown = compute_wfqs(entry.walk_forward_summary)
|
|
fresh_wfqs_v2_score, _ = compute_wfqs_v2(entry.walk_forward_summary)
|
|
fresh_deployment_score, fresh_deployment_breakdown = compute_deployment_score(
|
|
train_result,
|
|
valid_result,
|
|
test_result,
|
|
entry.walk_forward_summary,
|
|
rqs_score=fresh_rqs_score,
|
|
wfqs_score=fresh_wfqs_score,
|
|
)
|
|
public_sqs, public_breakdown, public_source = compute_public_sqs(
|
|
train_result,
|
|
valid_result,
|
|
test_result,
|
|
walk_forward_summary=entry.walk_forward_summary,
|
|
robustness_matrix_summary=entry.robustness_matrix_summary,
|
|
out_of_time_robustness_summary=entry.out_of_time_robustness_summary,
|
|
rqs_score=fresh_rqs_score,
|
|
wfqs_score=fresh_wfqs_v2_score,
|
|
)
|
|
stress_sqs, _, _ = compute_public_sqs_v2(
|
|
train_result,
|
|
valid_result,
|
|
test_result,
|
|
walk_forward_summary=entry.walk_forward_summary,
|
|
robustness_matrix_summary=entry.robustness_matrix_summary,
|
|
out_of_time_robustness_summary=entry.out_of_time_robustness_summary,
|
|
rqs_score=fresh_rqs_score,
|
|
wfqs_v2_score=fresh_wfqs_v2_score,
|
|
)
|
|
if public_sqs is None:
|
|
public_breakdown = public_breakdown or {}
|
|
public_source = public_source or "pending_validation"
|
|
|
|
print(f"\n{entry.entry_id} — {entry.experiment_name}")
|
|
print(f" Timestamp: {entry.timestamp}")
|
|
print(f" Hypothesis: {entry.hypothesis}")
|
|
print(f" Verdict: {entry.verdict}")
|
|
pending_reason = _validation_pending_reason(entry)
|
|
if public_sqs is None:
|
|
print(f" SQS: pending [{public_source}]")
|
|
if pending_reason:
|
|
print(f" Validation: {pending_reason}")
|
|
else:
|
|
print(f" SQS: {public_sqs} [{public_source}]")
|
|
if stress_sqs is not None:
|
|
print(f" Stress SQS: {stress_sqs}")
|
|
|
|
if entry.config_delta:
|
|
print(f" Baseline: {entry.config_delta.base_experiment}")
|
|
for key, value in entry.config_delta.changes.items():
|
|
print(f" {key}: {value}")
|
|
|
|
hydrated_results = {
|
|
"train": train_result,
|
|
"valid": valid_result,
|
|
"test": test_result,
|
|
**{
|
|
name: result
|
|
for name, result in entry.results.items()
|
|
if name not in {"train", "valid", "test"}
|
|
},
|
|
}
|
|
for split_name, split_result in _ordered_splits(hydrated_results):
|
|
pf = f"PF={split_result.profit_factor:.2f}" if split_result.profit_factor is not None else "PF=-"
|
|
ret = f"Ret={split_result.total_return_pct:+.2f}%" if split_result.total_return_pct is not None else "Ret=-"
|
|
ann = (
|
|
f"Ann={split_result.annualized_return_pct:+.2f}%"
|
|
if split_result.annualized_return_pct is not None
|
|
else "Ann=-"
|
|
)
|
|
dd = f"DD={split_result.max_drawdown_pct:.2f}%" if split_result.max_drawdown_pct is not None else "DD=-"
|
|
gross = (
|
|
f"Gross={split_result.avg_gross_exposure_pct:.2f}%"
|
|
if split_result.avg_gross_exposure_pct is not None
|
|
else "Gross=-"
|
|
)
|
|
dim = (
|
|
f"DIM={split_result.days_in_market_pct:.2f}%"
|
|
if split_result.days_in_market_pct is not None
|
|
else "DIM=-"
|
|
)
|
|
ret_on_gross = _ratio(split_result.total_return_pct, split_result.avg_gross_exposure_pct)
|
|
rog = f"R/G={ret_on_gross:.2f}" if ret_on_gross is not None else "R/G=-"
|
|
print(f" {split_name}: {split_result.trade_count} trades, {pf}, {ret}, {ann}, {dd}, {gross}, {dim}, {rog}")
|
|
|
|
if entry.walk_forward_summary is not None:
|
|
summary = entry.walk_forward_summary
|
|
test = summary.test_aggregate
|
|
gap = summary.gap_stats
|
|
print(
|
|
" WFV: "
|
|
f"{summary.fold_count} folds, "
|
|
f"mean={_fmt(test.mean_return_pct, '+.2f')}%, "
|
|
f"median={_fmt(test.median_return_pct, '+.2f')}%, "
|
|
f"worst={_fmt(test.worst_return_pct, '+.2f')}%, "
|
|
f"positive={_fmt(test.positive_fold_rate_pct, '.1f')}%"
|
|
)
|
|
print(
|
|
" WFV Gap: "
|
|
f"mean train-test gap={_fmt(gap.mean_train_test_return_gap_pct, '.2f')}%, "
|
|
f"worst={_fmt(gap.worst_train_test_return_gap_pct, '.2f')}%"
|
|
)
|
|
|
|
if entry.robustness_matrix_summary is not None:
|
|
summary = entry.robustness_matrix_summary
|
|
h63 = next((item for item in summary.horizon_summaries if item.horizon_days == 63), None)
|
|
h252 = next((item for item in summary.horizon_summaries if item.horizon_days == 252), None)
|
|
print(
|
|
" Robust: "
|
|
f"windows={summary.overall_window_count}, "
|
|
f"positive={_fmt(summary.overall_positive_window_rate_pct, '.1f')}%, "
|
|
f"worst={_fmt(summary.overall_worst_return_pct, '+.2f')}%, "
|
|
f"63d med={_fmt(h63.median_return_pct if h63 else None, '+.2f')}%, "
|
|
f"252d med={_fmt(h252.median_return_pct if h252 else None, '+.2f')}%"
|
|
)
|
|
for horizon in summary.horizon_summaries:
|
|
print(
|
|
" "
|
|
f"{horizon.horizon_days:>3}d: "
|
|
f"n={horizon.window_count}, "
|
|
f"mean={_fmt(horizon.mean_return_pct, '+.2f')}%, "
|
|
f"median={_fmt(horizon.median_return_pct, '+.2f')}%, "
|
|
f"worst={_fmt(horizon.worst_return_pct, '+.2f')}%, "
|
|
f"positive={_fmt(horizon.positive_window_rate_pct, '.1f')}%, "
|
|
f"dd={_fmt(horizon.mean_max_drawdown_pct, '.2f')}%"
|
|
)
|
|
if entry.out_of_time_robustness_summary is not None:
|
|
summary = entry.out_of_time_robustness_summary
|
|
h63 = next((item for item in summary.horizon_summaries if item.horizon_days == 63), None)
|
|
h252 = next((item for item in summary.horizon_summaries if item.horizon_days == 252), None)
|
|
print(
|
|
" OOT Robust: "
|
|
f"windows={summary.overall_window_count}, "
|
|
f"positive={_fmt(summary.overall_positive_window_rate_pct, '.1f')}%, "
|
|
f"worst={_fmt(summary.overall_worst_return_pct, '+.2f')}%, "
|
|
f"63d med={_fmt(h63.median_return_pct if h63 else None, '+.2f')}%, "
|
|
f"252d med={_fmt(h252.median_return_pct if h252 else None, '+.2f')}%"
|
|
)
|
|
|
|
if args.diagnostics:
|
|
promotion_score = entry.promotion_score
|
|
promotion_breakdown = entry.promotion_breakdown
|
|
if promotion_score is None:
|
|
promotion_score, promotion_breakdown = compute_promotion_score(
|
|
test_result,
|
|
valid_result,
|
|
)
|
|
unified_score = entry.unified_score
|
|
unified_breakdown = entry.unified_breakdown
|
|
if unified_score is None:
|
|
unified_score, unified_breakdown = compute_unified_score(
|
|
test_result,
|
|
valid_result,
|
|
)
|
|
rqs_score = fresh_rqs_score
|
|
rqs_breakdown = fresh_rqs_breakdown
|
|
wfqs_score = fresh_wfqs_score
|
|
wfqs_breakdown = fresh_wfqs_breakdown
|
|
deployment_score = fresh_deployment_score
|
|
deployment_breakdown = fresh_deployment_breakdown
|
|
robustness_gate_factor, robustness_breakdown = compute_robustness_gate(
|
|
entry.robustness_matrix_summary,
|
|
)
|
|
oot_gate_factor, oot_breakdown = compute_oot_robustness_gate(
|
|
entry.out_of_time_robustness_summary,
|
|
)
|
|
print(" Diagnostics:")
|
|
print(f" Public SQS: {public_sqs} {public_breakdown}")
|
|
if rqs_score is not None:
|
|
print(f" RQS: {rqs_score} {rqs_breakdown}")
|
|
if wfqs_score is not None:
|
|
print(f" WFQS: {wfqs_score} {wfqs_breakdown}")
|
|
wfqs_v2_score = entry.wfqs_v2_score
|
|
wfqs_v2_breakdown = entry.wfqs_v2_breakdown
|
|
if wfqs_v2_score is None:
|
|
wfqs_v2_score, wfqs_v2_breakdown = compute_wfqs_v2(entry.walk_forward_summary)
|
|
if wfqs_v2_score is not None:
|
|
print(f" WFQS v2: {wfqs_v2_score} {wfqs_v2_breakdown}")
|
|
if deployment_score is not None:
|
|
print(f" Deploy: {deployment_score} {deployment_breakdown}")
|
|
if promotion_score is not None:
|
|
print(f" Promotion: {promotion_score} {promotion_breakdown}")
|
|
if unified_score is not None:
|
|
print(f" Unified: {unified_score} {unified_breakdown}")
|
|
if entry.robustness_matrix_summary is not None:
|
|
print(f" RobustGate: {robustness_gate_factor:.2f} {robustness_breakdown}")
|
|
if entry.out_of_time_robustness_summary is not None:
|
|
print(f" OOTGate: {oot_gate_factor:.2f} {oot_breakdown}")
|
|
|
|
if entry.verdict_reasoning:
|
|
print(f" Reasoning: {entry.verdict_reasoning}")
|
|
if entry.next_direction:
|
|
print(f" Next: {entry.next_direction}")
|
|
|
|
|
|
def cmd_check_duplicate(args: argparse.Namespace) -> None:
|
|
"""Check if an experiment has been recorded already."""
|
|
journal_dir = Path(args.journal_dir)
|
|
journal_path = journal_dir / "improvement_journal.jsonl"
|
|
dupes = check_duplicate(journal_path, args.experiment)
|
|
if dupes:
|
|
print(f"FOUND {len(dupes)} existing entries for '{args.experiment}':")
|
|
for entry in dupes:
|
|
print(f" {entry.entry_id} ({entry.timestamp[:10]}) — SQS={entry.sqs_score}, verdict={entry.verdict}")
|
|
else:
|
|
print(f"No existing entries for '{args.experiment}'.")
|
|
|
|
|
|
def cmd_attach_wfv(args: argparse.Namespace) -> None:
|
|
"""Attach a walk-forward summary to an existing journal entry."""
|
|
journal_dir = Path(args.journal_dir)
|
|
journal_path = journal_dir / "improvement_journal.jsonl"
|
|
registry_path = journal_dir / "experiment_registry.json"
|
|
leaderboard_path = journal_dir / "LEADERBOARD.md"
|
|
summary_path = Path(args.summary)
|
|
|
|
if not journal_path.exists():
|
|
print("No journal found. Run 'record' first.")
|
|
sys.exit(1)
|
|
if not summary_path.exists():
|
|
print(f"ERROR: walk-forward summary not found: {summary_path}")
|
|
sys.exit(1)
|
|
|
|
summary = WalkForwardSummary.model_validate_json(summary_path.read_text())
|
|
with journal_lock(journal_path):
|
|
updated = attach_walk_forward_summary(journal_path, args.target, summary)
|
|
_sync_and_rebuild(journal_path, registry_path, leaderboard_path)
|
|
|
|
print(
|
|
f"Attached WFV to {updated.entry_id}: {updated.experiment_name} "
|
|
f"(WFQS={_fmt(updated.wfqs_score, '.1f')}, DEP={_fmt(updated.deployment_score, '.1f')}, folds={summary.fold_count})"
|
|
)
|
|
print(f"Leaderboard updated: {leaderboard_path}")
|
|
|
|
|
|
def cmd_attach_robustness(args: argparse.Namespace) -> None:
|
|
"""Attach a robustness matrix summary to an existing journal entry."""
|
|
journal_dir = Path(args.journal_dir)
|
|
journal_path = journal_dir / "improvement_journal.jsonl"
|
|
registry_path = journal_dir / "experiment_registry.json"
|
|
leaderboard_path = journal_dir / "LEADERBOARD.md"
|
|
summary_path = Path(args.summary)
|
|
|
|
if not journal_path.exists():
|
|
print("No journal found. Run 'record' first.")
|
|
sys.exit(1)
|
|
if not summary_path.exists():
|
|
print(f"ERROR: robustness summary not found: {summary_path}")
|
|
sys.exit(1)
|
|
|
|
summary = RobustnessMatrixSummary.model_validate_json(summary_path.read_text())
|
|
with journal_lock(journal_path):
|
|
updated = attach_robustness_summary(journal_path, args.target, summary)
|
|
_sync_and_rebuild(journal_path, registry_path, leaderboard_path)
|
|
|
|
gate_factor, _ = compute_robustness_gate(summary)
|
|
print(
|
|
f"Attached robustness matrix to {updated.entry_id}: {updated.experiment_name} "
|
|
f"(windows={summary.overall_window_count}, gate={gate_factor:.2f})"
|
|
)
|
|
print(f"Leaderboard updated: {leaderboard_path}")
|
|
|
|
|
|
def cmd_attach_oot_robustness(args: argparse.Namespace) -> None:
|
|
"""Attach out-of-time robustness matrix summary to an existing journal entry."""
|
|
journal_dir = Path(args.journal_dir)
|
|
journal_path = journal_dir / "improvement_journal.jsonl"
|
|
registry_path = journal_dir / "experiment_registry.json"
|
|
leaderboard_path = journal_dir / "LEADERBOARD.md"
|
|
summary_path = Path(args.summary)
|
|
|
|
if not journal_path.exists():
|
|
print("No journal found. Run 'record' first.")
|
|
sys.exit(1)
|
|
if not summary_path.exists():
|
|
print(f"ERROR: robustness summary not found: {summary_path}")
|
|
sys.exit(1)
|
|
|
|
summary = RobustnessMatrixSummary.model_validate_json(summary_path.read_text())
|
|
with journal_lock(journal_path):
|
|
updated = attach_out_of_time_robustness_summary(journal_path, args.target, summary)
|
|
_sync_and_rebuild(journal_path, registry_path, leaderboard_path)
|
|
|
|
gate_factor, _ = compute_oot_robustness_gate(summary)
|
|
print(
|
|
f"Attached out-of-time robustness to {updated.entry_id}: {updated.experiment_name} "
|
|
f"(windows={summary.overall_window_count}, gate={gate_factor:.2f})"
|
|
)
|
|
print(f"Leaderboard updated: {leaderboard_path}")
|
|
|
|
|
|
def cmd_rescore_public(args: argparse.Namespace) -> None:
|
|
"""Recompute stored public scores for matching journal entries."""
|
|
journal_dir = Path(args.journal_dir)
|
|
journal_path = journal_dir / "improvement_journal.jsonl"
|
|
registry_path = journal_dir / "experiment_registry.json"
|
|
leaderboard_path = journal_dir / "LEADERBOARD.md"
|
|
|
|
prefix = args.prefix or ""
|
|
|
|
def _matches(entry: JournalEntry) -> bool:
|
|
if prefix and not entry.experiment_name.startswith(prefix):
|
|
return False
|
|
if args.min_version is None and args.max_version is None:
|
|
return True
|
|
if not prefix:
|
|
return False
|
|
suffix = entry.experiment_name[len(prefix):]
|
|
try:
|
|
version = float(suffix)
|
|
except ValueError:
|
|
return False
|
|
if args.min_version is not None and version < args.min_version:
|
|
return False
|
|
if args.max_version is not None and version > args.max_version:
|
|
return False
|
|
return True
|
|
|
|
with journal_lock(journal_path):
|
|
updated_count = refresh_public_scores(journal_path, selector=_matches)
|
|
_sync_and_rebuild(journal_path, registry_path, leaderboard_path)
|
|
|
|
print(f"Rescored entries: {updated_count}")
|
|
print(f"Leaderboard updated: {leaderboard_path}")
|
|
|
|
|
|
def _print_help() -> None:
|
|
_console.print()
|
|
_console.print(Panel(
|
|
"[bold cyan]fithia2[/] — ACE-F Strategy Research & Paper Trading\n"
|
|
"[dim]백테스트 실험 추적 + Alpaca 페이퍼 트레이딩[/]",
|
|
border_style="cyan",
|
|
padding=(0, 2),
|
|
))
|
|
|
|
table = Table(box=box.SIMPLE, show_header=True, header_style="bold yellow", padding=(0, 2))
|
|
table.add_column("Command", style="bold green", no_wrap=True)
|
|
table.add_column("Description")
|
|
table.add_column("Key Options", style="dim")
|
|
table.add_row(
|
|
"leaderboard [dim]lb[/]",
|
|
"기본 public SQS 순위표 출력 및 LEADERBOARD.md 재생성",
|
|
"-n N --sort sqs|rqs|wfqs|deployment|promotion --include-retired",
|
|
)
|
|
table.add_row(
|
|
"record [dim]rec[/]",
|
|
"실험 결과를 저널에 기록",
|
|
"-e NAME -H TEXT --walk-forward-summary PATH --robustness-summary PATH --out-of-time-robustness-summary PATH",
|
|
)
|
|
table.add_row(
|
|
"show [dim]s[/]",
|
|
"특정 저널 항목 상세 조회",
|
|
"ENTRY_ID --diagnostics",
|
|
)
|
|
table.add_row(
|
|
"paper",
|
|
"Alpaca 페이퍼 트레이딩 [dim](fithia2 paper 로 상세 확인)[/]",
|
|
"start run positions status trades ...",
|
|
)
|
|
table.add_row(
|
|
"pipeline",
|
|
"데이터 파이프라인 실행 [dim](fithia2 pipeline 로 상세 확인)[/]",
|
|
"run [dim]--step poller|fetcher|parser|features|labels[/]",
|
|
)
|
|
_console.print(table)
|
|
|
|
_console.print(
|
|
" [dim]공통 옵션:[/] [bold]--journal-dir[/] [dim](기본: journal/)[/]"
|
|
" [bold]--runs-dir[/] [dim](기본: runs/)[/]\n"
|
|
)
|
|
_console.print(" [bold]예시[/]")
|
|
_console.print(" [green]fithia2 leaderboard[/]")
|
|
_console.print(" [green]fithia2 record --experiment pead_v2 --hypothesis '...' --verdict better[/]")
|
|
_console.print(" [green]fithia2 paper[/]")
|
|
_console.print()
|
|
|
|
|
|
def main() -> None:
|
|
# Delegate `fithia2 paper ...` to the paper trader CLI
|
|
if len(sys.argv) >= 2 and sys.argv[1] == "paper":
|
|
sys.argv = [sys.argv[0]] + sys.argv[2:]
|
|
from apps.paper_trader.cli import main as paper_main
|
|
paper_main()
|
|
return
|
|
|
|
# Delegate `fithia2 pipeline ...` to the pipeline CLI
|
|
if len(sys.argv) >= 2 and sys.argv[1] == "pipeline":
|
|
sys.argv = [sys.argv[0]] + sys.argv[2:]
|
|
from apps.pipeline.cli import main as pipeline_main
|
|
pipeline_main()
|
|
return
|
|
|
|
if len(sys.argv) == 1:
|
|
_print_help()
|
|
sys.exit(0)
|
|
|
|
parser = argparse.ArgumentParser(description="Strategy Improvement Tracker", add_help=True)
|
|
sub = parser.add_subparsers(dest="command", required=True)
|
|
|
|
journal_kwargs = {
|
|
"default": _DEFAULT_JOURNAL_DIR,
|
|
"help": f"Path to journal/ directory (default: {_DEFAULT_JOURNAL_DIR})",
|
|
}
|
|
|
|
for name in ("record", "rec"):
|
|
subparser = sub.add_parser(name, help="Record an experiment to the journal")
|
|
subparser.add_argument("--journal-dir", **journal_kwargs)
|
|
subparser.add_argument("--runs-dir", default="runs", help="Path to runs output directory (default: runs)")
|
|
subparser.add_argument("--experiment", "-e", required=True, help="Experiment name (matches manifest)")
|
|
subparser.add_argument("--hypothesis", "-H", help="What you expected this change to do")
|
|
subparser.add_argument("--baseline", "-b", help="Baseline experiment name for comparison")
|
|
subparser.add_argument("--verdict", "-v", choices=["better", "worse", "neutral", "unknown"], default="unknown")
|
|
subparser.add_argument("--reasoning", "-r", help="Why this verdict")
|
|
subparser.add_argument("--next", "-n", help="Next experiment direction")
|
|
subparser.add_argument("--walk-forward-summary", help="Optional path to walk_forward_summary.json")
|
|
subparser.add_argument("--robustness-summary", help="Optional path to robustness_matrix_summary.json")
|
|
subparser.add_argument(
|
|
"--out-of-time-robustness-summary",
|
|
help="Optional path to out-of-time robustness_matrix_summary.json",
|
|
)
|
|
subparser.add_argument("--force", "-f", action="store_true", help="Allow duplicate experiment names")
|
|
|
|
for name in ("leaderboard", "lb"):
|
|
subparser = sub.add_parser(name, help="Show/regenerate the leaderboard")
|
|
subparser.add_argument("--journal-dir", **journal_kwargs)
|
|
subparser.add_argument("--top", "-n", type=int, default=10, help="Show top N entries (default: 10)")
|
|
subparser.add_argument(
|
|
"--sort",
|
|
choices=["sqs", "rqs", "wfqs", "deployment", "dep", "promotion", "unified", "sqs2"],
|
|
default="sqs",
|
|
help="Sort by public SQS (default) or internal diagnostics",
|
|
)
|
|
subparser.add_argument(
|
|
"--include-retired",
|
|
action="store_true",
|
|
help="Include retired legacy PEAD / short-core / exact-pocket families",
|
|
)
|
|
|
|
for name in ("show", "s"):
|
|
subparser = sub.add_parser(name, help="Show details of a journal entry")
|
|
subparser.add_argument("--journal-dir", **journal_kwargs)
|
|
subparser.add_argument("entry_id", help="Entry ID (IMP-0001) or experiment name")
|
|
subparser.add_argument("--diagnostics", action="store_true", help="Show internal RQS/WFQS/DEP diagnostics")
|
|
|
|
for name in ("check-duplicate", "dup"):
|
|
subparser = sub.add_parser(name, help="Check if experiment already recorded")
|
|
subparser.add_argument("--journal-dir", **journal_kwargs)
|
|
subparser.add_argument("--experiment", "-e", required=True, help="Experiment name to check")
|
|
|
|
for name in ("attach-wfv", "awf"):
|
|
subparser = sub.add_parser(name, help="Attach walk-forward summary to an existing journal entry")
|
|
subparser.add_argument("--journal-dir", **journal_kwargs)
|
|
subparser.add_argument("target", help="Entry ID or experiment name to update")
|
|
subparser.add_argument("--summary", required=True, help="Path to walk_forward_summary.json")
|
|
|
|
for name in ("attach-robustness", "arb"):
|
|
subparser = sub.add_parser(name, help="Attach robustness matrix summary to an existing journal entry")
|
|
subparser.add_argument("--journal-dir", **journal_kwargs)
|
|
subparser.add_argument("target", help="Entry ID or experiment name to update")
|
|
subparser.add_argument("--summary", required=True, help="Path to robustness_matrix_summary.json")
|
|
|
|
for name in ("attach-oot-robustness", "aoot"):
|
|
subparser = sub.add_parser(name, help="Attach out-of-time robustness summary to an existing journal entry")
|
|
subparser.add_argument("--journal-dir", **journal_kwargs)
|
|
subparser.add_argument("target", help="Entry ID or experiment name to update")
|
|
subparser.add_argument("--summary", required=True, help="Path to robustness_matrix_summary.json")
|
|
|
|
for name in ("rescore-public", "rsp"):
|
|
subparser = sub.add_parser(name, help="Recompute stored public scores for journal entries")
|
|
subparser.add_argument("--journal-dir", **journal_kwargs)
|
|
subparser.add_argument("--prefix", help="Only rescore experiment names with this prefix")
|
|
subparser.add_argument("--min-version", type=float, help="Minimum numeric suffix after --prefix")
|
|
subparser.add_argument("--max-version", type=float, help="Maximum numeric suffix after --prefix")
|
|
|
|
args = parser.parse_args()
|
|
dispatch = {
|
|
"record": cmd_record,
|
|
"rec": cmd_record,
|
|
"leaderboard": cmd_leaderboard,
|
|
"lb": cmd_leaderboard,
|
|
"show": cmd_show,
|
|
"s": cmd_show,
|
|
"check-duplicate": cmd_check_duplicate,
|
|
"dup": cmd_check_duplicate,
|
|
"attach-wfv": cmd_attach_wfv,
|
|
"awf": cmd_attach_wfv,
|
|
"attach-robustness": cmd_attach_robustness,
|
|
"arb": cmd_attach_robustness,
|
|
"attach-oot-robustness": cmd_attach_oot_robustness,
|
|
"aoot": cmd_attach_oot_robustness,
|
|
"rescore-public": cmd_rescore_public,
|
|
"rsp": cmd_rescore_public,
|
|
}
|
|
dispatch[args.command](args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|