You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
512 lines
18 KiB
Python
512 lines
18 KiB
Python
"""CLI for experiment management.
|
|
|
|
Usage:
|
|
fithia2 exp create --parent NAME --name NAME [--changelog TEXT] [--created-by TEXT]
|
|
fithia2 exp search [--tag TAG] [--status STATUS] [--family FAMILY] [--parent NAME] [--pattern REGEX]
|
|
fithia2 exp tree ANCESTOR [--configs-dir DIR]
|
|
fithia2 exp info NAME
|
|
fithia2 exp diff NAME_A NAME_B
|
|
fithia2 exp promote NAME [--alias ALIAS]
|
|
fithia2 exp retire NAME
|
|
fithia2 exp validate [--fix]
|
|
fithia2 exp migrate [--dry-run]
|
|
fithia2 exp rebuild-index
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from rich import box
|
|
from rich.console import Console
|
|
from rich.panel import Panel
|
|
from rich.table import Table
|
|
|
|
_console = Console(width=140)
|
|
_EXPERIMENTS_DIR = Path("configs/experiments")
|
|
_JOURNAL_PATH = Path("journal/improvement_journal.jsonl")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _exp_lib():
|
|
from libs.backtest import experiments as exp_lib
|
|
return exp_lib
|
|
|
|
|
|
def _resolve(id_or_name: str) -> str:
|
|
"""Resolve numeric ID or experiment name to canonical name. Exits on failure."""
|
|
try:
|
|
return _exp_lib().resolve_experiment_name(id_or_name, _EXPERIMENTS_DIR)
|
|
except KeyError as e:
|
|
_console.print(f"[red]Error:[/] {e}")
|
|
sys.exit(1)
|
|
|
|
|
|
def _status_style(status: str) -> str:
|
|
return {
|
|
"promoted": "bold green",
|
|
"active": "cyan",
|
|
"draft": "yellow",
|
|
"retired": "dim",
|
|
}.get(status, "white")
|
|
|
|
|
|
def _sqs_str(sqs: float | None) -> str:
|
|
if sqs is None:
|
|
return "[dim]—[/]"
|
|
color = "green" if sqs >= 70 else ("yellow" if sqs >= 60 else "red")
|
|
return f"[{color}]{sqs:.1f}[/]"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Subcommand handlers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def cmd_create(args: argparse.Namespace) -> int:
|
|
lib = _exp_lib()
|
|
try:
|
|
path = lib.create_experiment(
|
|
parent_name=_resolve(args.parent),
|
|
new_name=args.name,
|
|
changelog=args.changelog,
|
|
created_by=args.created_by,
|
|
configs_dir=_EXPERIMENTS_DIR,
|
|
)
|
|
_console.print(f"[green]✓ Created:[/] {path}")
|
|
_console.print(f" parent: [cyan]{args.parent}[/]")
|
|
if args.changelog:
|
|
_console.print(f" changelog: {args.changelog}")
|
|
_console.print(f" status: [yellow]draft[/] — run backtest then [bold]fithia2 exp promote {args.name}[/] when ready")
|
|
return 0
|
|
except FileNotFoundError as e:
|
|
_console.print(f"[red]Error:[/] {e}")
|
|
return 1
|
|
except FileExistsError as e:
|
|
_console.print(f"[red]Error:[/] {e}")
|
|
return 1
|
|
|
|
|
|
def cmd_search(args: argparse.Namespace) -> int:
|
|
lib = _exp_lib()
|
|
journal = _JOURNAL_PATH if _JOURNAL_PATH.exists() else None
|
|
results = lib.search_experiments(
|
|
configs_dir=_EXPERIMENTS_DIR,
|
|
journal_path=journal,
|
|
tag=args.tag,
|
|
status=args.status,
|
|
version_family=args.family,
|
|
parent=args.parent,
|
|
name_pattern=args.pattern,
|
|
has_journal_entry=None,
|
|
)
|
|
|
|
if not results:
|
|
_console.print("[dim]No experiments matched the search criteria.[/]")
|
|
return 0
|
|
|
|
table = Table(box=box.SIMPLE, show_header=True, header_style="bold yellow", padding=(0, 1))
|
|
table.add_column("ID", justify="right", style="bold dim", no_wrap=True)
|
|
table.add_column("Name", style="cyan", no_wrap=True)
|
|
table.add_column("Status", no_wrap=True)
|
|
table.add_column("Family", style="dim")
|
|
table.add_column("Gen", justify="right", style="dim")
|
|
table.add_column("Parent", style="dim", no_wrap=True)
|
|
table.add_column("SQS", justify="right")
|
|
table.add_column("Aliases", style="dim")
|
|
table.add_column("Tags", style="dim")
|
|
|
|
for r in sorted(results, key=lambda x: x.get("id") or 0):
|
|
status = r.get("status", "active")
|
|
aliases = ", ".join(r.get("aliases") or [])
|
|
tags = ", ".join(r.get("tags") or [])[:40]
|
|
gen = str(r.get("generation")) if r.get("generation") is not None else "—"
|
|
parent = r.get("parent") or "—"
|
|
# Shorten parent name for display
|
|
if parent and parent.startswith("return_max_long_"):
|
|
parent = parent[len("return_max_long_"):]
|
|
eid = str(r.get("id")) if r.get("id") is not None else "—"
|
|
table.add_row(
|
|
eid,
|
|
r["name"],
|
|
f"[{_status_style(status)}]{status}[/]",
|
|
r.get("version_family") or "—",
|
|
gen,
|
|
parent,
|
|
_sqs_str(r.get("sqs_score")),
|
|
aliases or "—",
|
|
tags or "—",
|
|
)
|
|
|
|
_console.print(f"\n[bold]Found {len(results)} experiment(s)[/]\n")
|
|
_console.print(table)
|
|
return 0
|
|
|
|
|
|
def cmd_tree(args: argparse.Namespace) -> int:
|
|
lib = _exp_lib()
|
|
ancestor = _resolve(args.ancestor)
|
|
journal = _JOURNAL_PATH if _JOURNAL_PATH.exists() else None
|
|
try:
|
|
tree = lib.build_lineage_tree(
|
|
ancestor_name=ancestor,
|
|
configs_dir=_EXPERIMENTS_DIR,
|
|
journal_path=journal,
|
|
)
|
|
except KeyError as e:
|
|
_console.print(f"[red]Error:[/] {e}")
|
|
return 1
|
|
|
|
rendered = lib.format_tree(tree)
|
|
_console.print(Panel(rendered, title=f"[bold cyan]Lineage Tree: {ancestor}[/]", border_style="cyan"))
|
|
return 0
|
|
|
|
|
|
def cmd_info(args: argparse.Namespace) -> int:
|
|
lib = _exp_lib()
|
|
name = _resolve(args.name)
|
|
path = _EXPERIMENTS_DIR / f"{name}.json"
|
|
if not path.exists():
|
|
_console.print(f"[red]Error:[/] Experiment not found: {name}")
|
|
return 1
|
|
|
|
import json
|
|
data = json.loads(path.read_text())
|
|
|
|
# Cross-reference journal
|
|
journal = _JOURNAL_PATH if _JOURNAL_PATH.exists() else None
|
|
journal_entry: dict | None = None
|
|
if journal:
|
|
for line in journal.read_text().splitlines():
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
try:
|
|
e = json.loads(line)
|
|
if e.get("experiment_name") == name:
|
|
journal_entry = e
|
|
except Exception:
|
|
pass
|
|
|
|
table = Table(box=box.MINIMAL, show_header=False, padding=(0, 1))
|
|
table.add_column("Key", style="bold cyan", no_wrap=True)
|
|
table.add_column("Value")
|
|
|
|
meta_rows = [
|
|
("id", str(data["id"]) if data.get("id") is not None else "—"),
|
|
("experiment_name", data.get("experiment_name")),
|
|
("status", data.get("status", "active")),
|
|
("version_family", data.get("version_family") or "—"),
|
|
("generation", str(data.get("generation")) if data.get("generation") is not None else "—"),
|
|
("parent", data.get("parent") or "—"),
|
|
("created_at", data.get("created_at") or "—"),
|
|
("created_by", data.get("created_by") or "—"),
|
|
("description", data.get("description") or "—"),
|
|
("changelog", data.get("changelog") or "—"),
|
|
("tags", ", ".join(data.get("tags") or [])),
|
|
("aliases", ", ".join(data.get("aliases") or [])),
|
|
("dataset_snapshot_id", data.get("dataset_snapshot_id")),
|
|
]
|
|
|
|
for k, v in meta_rows:
|
|
table.add_row(k, str(v) if v else "—")
|
|
|
|
if journal_entry:
|
|
table.add_row("", "")
|
|
table.add_row("[bold yellow]Journal[/]", "")
|
|
table.add_row("entry_id", journal_entry.get("entry_id", "—"))
|
|
sqs = journal_entry.get("sqs_score")
|
|
table.add_row("sqs_score", f"{sqs:.2f}" if sqs is not None else "—")
|
|
rqs = journal_entry.get("rqs_score")
|
|
table.add_row("rqs_score", f"{rqs:.2f}" if rqs is not None else "—")
|
|
public_sqs = journal_entry.get("sqs_v3_score")
|
|
table.add_row("public_sqs", f"{public_sqs:.2f}" if public_sqs is not None else "—")
|
|
table.add_row("verdict", journal_entry.get("verdict", "—"))
|
|
elif data.get("performance_summary"):
|
|
table.add_row("", "")
|
|
table.add_row("[bold yellow]Cached Performance[/]", "")
|
|
for k, v in data["performance_summary"].items():
|
|
table.add_row(f" {k}", str(v))
|
|
|
|
# Ancestry chain
|
|
chain = lib.get_ancestor_chain(name, configs_dir=_EXPERIMENTS_DIR)
|
|
if len(chain) > 1:
|
|
table.add_row("", "")
|
|
table.add_row("[bold yellow]Ancestry[/]", "")
|
|
for i, ancestor in enumerate(chain):
|
|
eid = f"#{ancestor['id']}" if ancestor["id"] is not None else " "
|
|
is_self = ancestor["name"] == name
|
|
indent = " " * i
|
|
cl = ancestor.get("changelog") or ancestor.get("description") or ""
|
|
cl_short = cl[:50] + "…" if len(cl) > 50 else cl
|
|
if is_self:
|
|
label = f"[bold cyan]{indent}▶ {eid} {ancestor['name']}[/]"
|
|
else:
|
|
label = f"{indent} {eid} [dim]{ancestor['name']}[/]"
|
|
table.add_row(label, f"[dim]{cl_short}[/]" if cl_short else "")
|
|
|
|
_console.print(Panel(table, title=f"[bold cyan]{name}[/]", border_style="cyan"))
|
|
return 0
|
|
|
|
|
|
def cmd_diff(args: argparse.Namespace) -> int:
|
|
lib = _exp_lib()
|
|
name_a = _resolve(args.name_a)
|
|
name_b = _resolve(args.name_b)
|
|
try:
|
|
diffs = lib.diff_experiments(name_a, name_b, configs_dir=_EXPERIMENTS_DIR)
|
|
except FileNotFoundError as e:
|
|
_console.print(f"[red]Error:[/] {e}")
|
|
return 1
|
|
|
|
if not diffs:
|
|
_console.print(f"[green]✓ No differences found between {name_a} and {name_b}[/]")
|
|
return 0
|
|
|
|
table = Table(box=box.SIMPLE, show_header=True, header_style="bold yellow", padding=(0, 1))
|
|
table.add_column("Path", style="cyan", no_wrap=True)
|
|
table.add_column(name_a, style="red")
|
|
table.add_column(name_b, style="green")
|
|
|
|
for path, (va, vb) in sorted(diffs.items()):
|
|
va_str = _format_diff_value(va)
|
|
vb_str = _format_diff_value(vb)
|
|
table.add_row(path, va_str, vb_str)
|
|
|
|
_console.print(f"\n[bold]{len(diffs)} difference(s) found[/]\n")
|
|
_console.print(table)
|
|
return 0
|
|
|
|
|
|
def _format_diff_value(v) -> str:
|
|
if v is None:
|
|
return "[dim]∅[/]"
|
|
if isinstance(v, dict):
|
|
import json
|
|
s = json.dumps(v, ensure_ascii=False)
|
|
return s[:80] + "…" if len(s) > 80 else s
|
|
return str(v)
|
|
|
|
|
|
def cmd_promote(args: argparse.Namespace) -> int:
|
|
lib = _exp_lib()
|
|
name = _resolve(args.name)
|
|
path = _EXPERIMENTS_DIR / f"{name}.json"
|
|
if not path.exists():
|
|
_console.print(f"[red]Error:[/] Experiment not found: {name}")
|
|
return 1
|
|
|
|
lib.set_experiment_status(name, "promoted", configs_dir=_EXPERIMENTS_DIR)
|
|
_console.print(f"[green]✓ Promoted:[/] {name}")
|
|
|
|
# Optionally add alias
|
|
if args.alias:
|
|
import json
|
|
data = json.loads(path.read_text())
|
|
aliases = data.get("aliases") or []
|
|
if args.alias not in aliases:
|
|
aliases.append(args.alias)
|
|
data["aliases"] = aliases
|
|
path.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n")
|
|
_console.print(f" alias: [bold]{args.alias}[/] added")
|
|
|
|
return 0
|
|
|
|
|
|
def cmd_retire(args: argparse.Namespace) -> int:
|
|
lib = _exp_lib()
|
|
name = _resolve(args.name)
|
|
path = _EXPERIMENTS_DIR / f"{name}.json"
|
|
if not path.exists():
|
|
_console.print(f"[red]Error:[/] Experiment not found: {name}")
|
|
return 1
|
|
|
|
lib.set_experiment_status(name, "retired", configs_dir=_EXPERIMENTS_DIR)
|
|
_console.print(f"[dim]✓ Retired:[/] {name}")
|
|
return 0
|
|
|
|
|
|
def cmd_validate(args: argparse.Namespace) -> int:
|
|
lib = _exp_lib()
|
|
issues = lib.validate_all_experiments(configs_dir=_EXPERIMENTS_DIR)
|
|
|
|
if not issues:
|
|
_console.print("[green]✓ All experiments are valid.[/]")
|
|
return 0
|
|
|
|
table = Table(box=box.SIMPLE, show_header=True, header_style="bold yellow", padding=(0, 1))
|
|
table.add_column("File", style="cyan", no_wrap=True)
|
|
table.add_column("Issue", style="red")
|
|
|
|
for issue in issues:
|
|
table.add_row(issue["file"], issue["issue"])
|
|
|
|
_console.print(f"\n[bold red]{len(issues)} issue(s) found[/]\n")
|
|
_console.print(table)
|
|
|
|
if getattr(args, "fix", False):
|
|
_console.print("\n[yellow]--fix is not yet implemented. Please fix issues manually.[/]")
|
|
|
|
return 1 if issues else 0
|
|
|
|
|
|
def cmd_migrate(args: argparse.Namespace) -> int:
|
|
lib = _exp_lib()
|
|
dry_run = getattr(args, "dry_run", False)
|
|
if dry_run:
|
|
_console.print("[yellow]Dry run mode — no files will be modified[/]\n")
|
|
|
|
actions = lib.migrate_experiments(configs_dir=_EXPERIMENTS_DIR, dry_run=dry_run)
|
|
|
|
if not actions:
|
|
_console.print("[green]✓ All experiments already have metadata. Nothing to migrate.[/]")
|
|
return 0
|
|
|
|
table = Table(box=box.SIMPLE, show_header=True, header_style="bold yellow", padding=(0, 1))
|
|
table.add_column("Experiment", style="cyan", no_wrap=True)
|
|
table.add_column("Fields Added/Updated", style="dim")
|
|
|
|
for action in actions:
|
|
changes_str = ", ".join(
|
|
f"{k}={v!r}" if len(str(v)) < 30 else f"{k}=…"
|
|
for k, v in action["changes"].items()
|
|
)
|
|
table.add_row(action["name"], changes_str)
|
|
|
|
verb = "Would update" if dry_run else "Updated"
|
|
_console.print(f"[bold]{verb} {len(actions)} experiment(s)[/]\n")
|
|
_console.print(table)
|
|
|
|
if not dry_run:
|
|
_console.print("\n[green]✓ Migration complete. Index rebuilt.[/]")
|
|
|
|
return 0
|
|
|
|
|
|
def cmd_rebuild_index(args: argparse.Namespace) -> int:
|
|
lib = _exp_lib()
|
|
journal = _JOURNAL_PATH if _JOURNAL_PATH.exists() else None
|
|
index = lib.rebuild_experiment_index(configs_dir=_EXPERIMENTS_DIR, journal_path=journal)
|
|
count = len(index.get("experiments", {}))
|
|
_console.print(f"[green]✓ Index rebuilt:[/] {count} experiments indexed at {_EXPERIMENTS_DIR / '.index.json'}")
|
|
return 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CLI wiring
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _print_help() -> None:
|
|
_console.print(Panel(
|
|
"[bold cyan]fithia2 exp[/] — Experiment Management\n"
|
|
"[dim]전략 실험 생성, 검색, 계보 추적, 비교[/]",
|
|
border_style="cyan",
|
|
padding=(0, 2),
|
|
))
|
|
table = Table(box=box.SIMPLE, show_header=True, header_style="bold yellow", padding=(0, 2))
|
|
table.add_column("Command", style="bold green", no_wrap=True)
|
|
table.add_column("Description")
|
|
table.add_column("Key Options", style="dim")
|
|
table.add_row("create", "부모에서 새 실험 생성", "--parent NAME --name NAME [--changelog TEXT] [--created-by TEXT]")
|
|
table.add_row("search", "실험 검색", "[--tag TAG] [--status STATUS] [--family FAMILY] [--parent NAME] [--pattern REGEX]")
|
|
table.add_row("tree", "계보 트리 시각화", "ANCESTOR")
|
|
table.add_row("info", "단일 실험 상세 조회", "NAME")
|
|
table.add_row("diff", "두 실험 설정 비교", "NAME_A NAME_B")
|
|
table.add_row("promote", "status → promoted", "NAME [--alias ALIAS]")
|
|
table.add_row("retire", "status → retired", "NAME")
|
|
table.add_row("validate", "전체 스키마 검증", "[--fix]")
|
|
table.add_row("migrate", "기존 파일 메타데이터 백필", "[--dry-run]")
|
|
table.add_row("rebuild-index", "인덱스 캐시 재생성", "")
|
|
_console.print(table)
|
|
|
|
|
|
def main() -> None:
|
|
if len(sys.argv) < 2 or sys.argv[1] in ("-h", "--help"):
|
|
_print_help()
|
|
return
|
|
|
|
subcmd = sys.argv[1]
|
|
rest = sys.argv[2:]
|
|
|
|
if subcmd == "create":
|
|
parser = argparse.ArgumentParser(prog="fithia2 exp create")
|
|
parser.add_argument("--parent", "-p", required=True, help="Parent experiment name")
|
|
parser.add_argument("--name", "-n", required=True, help="New experiment name")
|
|
parser.add_argument("--changelog", "-c", help="Description of changes from parent")
|
|
parser.add_argument("--created-by", default="ai_agent", help="Creator identifier")
|
|
args = parser.parse_args(rest)
|
|
sys.exit(cmd_create(args))
|
|
|
|
elif subcmd == "search":
|
|
parser = argparse.ArgumentParser(prog="fithia2 exp search")
|
|
parser.add_argument("--tag", help="Filter by tag")
|
|
parser.add_argument("--status", help="Filter by status (draft|active|promoted|retired)")
|
|
parser.add_argument("--family", help="Filter by version_family (e.g. v6new, v8)")
|
|
parser.add_argument("--parent", help="Filter by parent experiment name")
|
|
parser.add_argument("--pattern", help="Filter by name regex pattern")
|
|
args = parser.parse_args(rest)
|
|
sys.exit(cmd_search(args))
|
|
|
|
elif subcmd == "tree":
|
|
parser = argparse.ArgumentParser(prog="fithia2 exp tree")
|
|
parser.add_argument("ancestor", help="Root experiment for the tree")
|
|
args = parser.parse_args(rest)
|
|
sys.exit(cmd_tree(args))
|
|
|
|
elif subcmd == "info":
|
|
parser = argparse.ArgumentParser(prog="fithia2 exp info")
|
|
parser.add_argument("name", help="Experiment name")
|
|
args = parser.parse_args(rest)
|
|
sys.exit(cmd_info(args))
|
|
|
|
elif subcmd == "diff":
|
|
parser = argparse.ArgumentParser(prog="fithia2 exp diff")
|
|
parser.add_argument("name_a", help="First experiment name")
|
|
parser.add_argument("name_b", help="Second experiment name")
|
|
args = parser.parse_args(rest)
|
|
sys.exit(cmd_diff(args))
|
|
|
|
elif subcmd == "promote":
|
|
parser = argparse.ArgumentParser(prog="fithia2 exp promote")
|
|
parser.add_argument("name", help="Experiment name")
|
|
parser.add_argument("--alias", help="Human-readable alias to add")
|
|
args = parser.parse_args(rest)
|
|
sys.exit(cmd_promote(args))
|
|
|
|
elif subcmd == "retire":
|
|
parser = argparse.ArgumentParser(prog="fithia2 exp retire")
|
|
parser.add_argument("name", help="Experiment name")
|
|
args = parser.parse_args(rest)
|
|
sys.exit(cmd_retire(args))
|
|
|
|
elif subcmd == "validate":
|
|
parser = argparse.ArgumentParser(prog="fithia2 exp validate")
|
|
parser.add_argument("--fix", action="store_true", help="Auto-fix fixable issues")
|
|
args = parser.parse_args(rest)
|
|
sys.exit(cmd_validate(args))
|
|
|
|
elif subcmd == "migrate":
|
|
parser = argparse.ArgumentParser(prog="fithia2 exp migrate")
|
|
parser.add_argument("--dry-run", action="store_true", help="Report changes without writing")
|
|
args = parser.parse_args(rest)
|
|
sys.exit(cmd_migrate(args))
|
|
|
|
elif subcmd in ("rebuild-index", "rebuild_index"):
|
|
args = argparse.Namespace()
|
|
sys.exit(cmd_rebuild_index(args))
|
|
|
|
else:
|
|
_console.print(f"[red]Unknown subcommand:[/] {subcmd!r}")
|
|
_print_help()
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|