"""CLI for experiment management. Usage: fithia2 exp create --parent NAME --name NAME [--changelog TEXT] [--created-by TEXT] fithia2 exp search [--tag TAG] [--status STATUS] [--family FAMILY] [--parent NAME] [--pattern REGEX] fithia2 exp tree ANCESTOR [--configs-dir DIR] fithia2 exp info NAME fithia2 exp diff NAME_A NAME_B fithia2 exp promote NAME [--alias ALIAS] fithia2 exp retire NAME fithia2 exp validate [--fix] fithia2 exp migrate [--dry-run] fithia2 exp rebuild-index """ from __future__ import annotations import argparse import sys from pathlib import Path from rich import box from rich.console import Console from rich.panel import Panel from rich.table import Table _console = Console(width=140) _EXPERIMENTS_DIR = Path("configs/experiments") _JOURNAL_PATH = Path("journal/improvement_journal.jsonl") # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def _exp_lib(): from libs.backtest import experiments as exp_lib return exp_lib def _resolve(id_or_name: str) -> str: """Resolve numeric ID or experiment name to canonical name. Exits on failure.""" try: return _exp_lib().resolve_experiment_name(id_or_name, _EXPERIMENTS_DIR) except KeyError as e: _console.print(f"[red]Error:[/] {e}") sys.exit(1) def _status_style(status: str) -> str: return { "promoted": "bold green", "active": "cyan", "draft": "yellow", "retired": "dim", }.get(status, "white") def _sqs_str(sqs: float | None) -> str: if sqs is None: return "[dim]—[/]" color = "green" if sqs >= 70 else ("yellow" if sqs >= 60 else "red") return f"[{color}]{sqs:.1f}[/]" # --------------------------------------------------------------------------- # Subcommand handlers # --------------------------------------------------------------------------- def cmd_create(args: argparse.Namespace) -> int: lib = _exp_lib() try: path = lib.create_experiment( parent_name=_resolve(args.parent), new_name=args.name, changelog=args.changelog, created_by=args.created_by, configs_dir=_EXPERIMENTS_DIR, ) _console.print(f"[green]✓ Created:[/] {path}") _console.print(f" parent: [cyan]{args.parent}[/]") if args.changelog: _console.print(f" changelog: {args.changelog}") _console.print(f" status: [yellow]draft[/] — run backtest then [bold]fithia2 exp promote {args.name}[/] when ready") return 0 except FileNotFoundError as e: _console.print(f"[red]Error:[/] {e}") return 1 except FileExistsError as e: _console.print(f"[red]Error:[/] {e}") return 1 def cmd_search(args: argparse.Namespace) -> int: lib = _exp_lib() journal = _JOURNAL_PATH if _JOURNAL_PATH.exists() else None results = lib.search_experiments( configs_dir=_EXPERIMENTS_DIR, journal_path=journal, tag=args.tag, status=args.status, version_family=args.family, parent=args.parent, name_pattern=args.pattern, has_journal_entry=None, ) if not results: _console.print("[dim]No experiments matched the search criteria.[/]") return 0 table = Table(box=box.SIMPLE, show_header=True, header_style="bold yellow", padding=(0, 1)) table.add_column("ID", justify="right", style="bold dim", no_wrap=True) table.add_column("Name", style="cyan", no_wrap=True) table.add_column("Status", no_wrap=True) table.add_column("Family", style="dim") table.add_column("Gen", justify="right", style="dim") table.add_column("Parent", style="dim", no_wrap=True) table.add_column("SQS", justify="right") table.add_column("Aliases", style="dim") table.add_column("Tags", style="dim") for r in sorted(results, key=lambda x: x.get("id") or 0): status = r.get("status", "active") aliases = ", ".join(r.get("aliases") or []) tags = ", ".join(r.get("tags") or [])[:40] gen = str(r.get("generation")) if r.get("generation") is not None else "—" parent = r.get("parent") or "—" # Shorten parent name for display if parent and parent.startswith("return_max_long_"): parent = parent[len("return_max_long_"):] eid = str(r.get("id")) if r.get("id") is not None else "—" table.add_row( eid, r["name"], f"[{_status_style(status)}]{status}[/]", r.get("version_family") or "—", gen, parent, _sqs_str(r.get("sqs_score")), aliases or "—", tags or "—", ) _console.print(f"\n[bold]Found {len(results)} experiment(s)[/]\n") _console.print(table) return 0 def cmd_tree(args: argparse.Namespace) -> int: lib = _exp_lib() ancestor = _resolve(args.ancestor) journal = _JOURNAL_PATH if _JOURNAL_PATH.exists() else None try: tree = lib.build_lineage_tree( ancestor_name=ancestor, configs_dir=_EXPERIMENTS_DIR, journal_path=journal, ) except KeyError as e: _console.print(f"[red]Error:[/] {e}") return 1 rendered = lib.format_tree(tree) _console.print(Panel(rendered, title=f"[bold cyan]Lineage Tree: {ancestor}[/]", border_style="cyan")) return 0 def cmd_info(args: argparse.Namespace) -> int: lib = _exp_lib() name = _resolve(args.name) path = _EXPERIMENTS_DIR / f"{name}.json" if not path.exists(): _console.print(f"[red]Error:[/] Experiment not found: {name}") return 1 import json data = json.loads(path.read_text()) # Cross-reference journal journal = _JOURNAL_PATH if _JOURNAL_PATH.exists() else None journal_entry: dict | None = None if journal: for line in journal.read_text().splitlines(): line = line.strip() if not line: continue try: e = json.loads(line) if e.get("experiment_name") == name: journal_entry = e except Exception: pass table = Table(box=box.MINIMAL, show_header=False, padding=(0, 1)) table.add_column("Key", style="bold cyan", no_wrap=True) table.add_column("Value") meta_rows = [ ("id", str(data["id"]) if data.get("id") is not None else "—"), ("experiment_name", data.get("experiment_name")), ("status", data.get("status", "active")), ("version_family", data.get("version_family") or "—"), ("generation", str(data.get("generation")) if data.get("generation") is not None else "—"), ("parent", data.get("parent") or "—"), ("created_at", data.get("created_at") or "—"), ("created_by", data.get("created_by") or "—"), ("description", data.get("description") or "—"), ("changelog", data.get("changelog") or "—"), ("tags", ", ".join(data.get("tags") or [])), ("aliases", ", ".join(data.get("aliases") or [])), ("dataset_snapshot_id", data.get("dataset_snapshot_id")), ] for k, v in meta_rows: table.add_row(k, str(v) if v else "—") if journal_entry: table.add_row("", "") table.add_row("[bold yellow]Journal[/]", "") table.add_row("entry_id", journal_entry.get("entry_id", "—")) sqs = journal_entry.get("sqs_score") table.add_row("sqs_score", f"{sqs:.2f}" if sqs is not None else "—") rqs = journal_entry.get("rqs_score") table.add_row("rqs_score", f"{rqs:.2f}" if rqs is not None else "—") public_sqs = journal_entry.get("sqs_v3_score") table.add_row("public_sqs", f"{public_sqs:.2f}" if public_sqs is not None else "—") table.add_row("verdict", journal_entry.get("verdict", "—")) elif data.get("performance_summary"): table.add_row("", "") table.add_row("[bold yellow]Cached Performance[/]", "") for k, v in data["performance_summary"].items(): table.add_row(f" {k}", str(v)) # Ancestry chain chain = lib.get_ancestor_chain(name, configs_dir=_EXPERIMENTS_DIR) if len(chain) > 1: table.add_row("", "") table.add_row("[bold yellow]Ancestry[/]", "") for i, ancestor in enumerate(chain): eid = f"#{ancestor['id']}" if ancestor["id"] is not None else " " is_self = ancestor["name"] == name indent = " " * i cl = ancestor.get("changelog") or ancestor.get("description") or "" cl_short = cl[:50] + "…" if len(cl) > 50 else cl if is_self: label = f"[bold cyan]{indent}▶ {eid} {ancestor['name']}[/]" else: label = f"{indent} {eid} [dim]{ancestor['name']}[/]" table.add_row(label, f"[dim]{cl_short}[/]" if cl_short else "") _console.print(Panel(table, title=f"[bold cyan]{name}[/]", border_style="cyan")) return 0 def cmd_diff(args: argparse.Namespace) -> int: lib = _exp_lib() name_a = _resolve(args.name_a) name_b = _resolve(args.name_b) try: diffs = lib.diff_experiments(name_a, name_b, configs_dir=_EXPERIMENTS_DIR) except FileNotFoundError as e: _console.print(f"[red]Error:[/] {e}") return 1 if not diffs: _console.print(f"[green]✓ No differences found between {name_a} and {name_b}[/]") return 0 table = Table(box=box.SIMPLE, show_header=True, header_style="bold yellow", padding=(0, 1)) table.add_column("Path", style="cyan", no_wrap=True) table.add_column(name_a, style="red") table.add_column(name_b, style="green") for path, (va, vb) in sorted(diffs.items()): va_str = _format_diff_value(va) vb_str = _format_diff_value(vb) table.add_row(path, va_str, vb_str) _console.print(f"\n[bold]{len(diffs)} difference(s) found[/]\n") _console.print(table) return 0 def _format_diff_value(v) -> str: if v is None: return "[dim]∅[/]" if isinstance(v, dict): import json s = json.dumps(v, ensure_ascii=False) return s[:80] + "…" if len(s) > 80 else s return str(v) def cmd_promote(args: argparse.Namespace) -> int: lib = _exp_lib() name = _resolve(args.name) path = _EXPERIMENTS_DIR / f"{name}.json" if not path.exists(): _console.print(f"[red]Error:[/] Experiment not found: {name}") return 1 lib.set_experiment_status(name, "promoted", configs_dir=_EXPERIMENTS_DIR) _console.print(f"[green]✓ Promoted:[/] {name}") # Optionally add alias if args.alias: import json data = json.loads(path.read_text()) aliases = data.get("aliases") or [] if args.alias not in aliases: aliases.append(args.alias) data["aliases"] = aliases path.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n") _console.print(f" alias: [bold]{args.alias}[/] added") return 0 def cmd_retire(args: argparse.Namespace) -> int: lib = _exp_lib() name = _resolve(args.name) path = _EXPERIMENTS_DIR / f"{name}.json" if not path.exists(): _console.print(f"[red]Error:[/] Experiment not found: {name}") return 1 lib.set_experiment_status(name, "retired", configs_dir=_EXPERIMENTS_DIR) _console.print(f"[dim]✓ Retired:[/] {name}") return 0 def cmd_validate(args: argparse.Namespace) -> int: lib = _exp_lib() issues = lib.validate_all_experiments(configs_dir=_EXPERIMENTS_DIR) if not issues: _console.print("[green]✓ All experiments are valid.[/]") return 0 table = Table(box=box.SIMPLE, show_header=True, header_style="bold yellow", padding=(0, 1)) table.add_column("File", style="cyan", no_wrap=True) table.add_column("Issue", style="red") for issue in issues: table.add_row(issue["file"], issue["issue"]) _console.print(f"\n[bold red]{len(issues)} issue(s) found[/]\n") _console.print(table) if getattr(args, "fix", False): _console.print("\n[yellow]--fix is not yet implemented. Please fix issues manually.[/]") return 1 if issues else 0 def cmd_migrate(args: argparse.Namespace) -> int: lib = _exp_lib() dry_run = getattr(args, "dry_run", False) if dry_run: _console.print("[yellow]Dry run mode — no files will be modified[/]\n") actions = lib.migrate_experiments(configs_dir=_EXPERIMENTS_DIR, dry_run=dry_run) if not actions: _console.print("[green]✓ All experiments already have metadata. Nothing to migrate.[/]") return 0 table = Table(box=box.SIMPLE, show_header=True, header_style="bold yellow", padding=(0, 1)) table.add_column("Experiment", style="cyan", no_wrap=True) table.add_column("Fields Added/Updated", style="dim") for action in actions: changes_str = ", ".join( f"{k}={v!r}" if len(str(v)) < 30 else f"{k}=…" for k, v in action["changes"].items() ) table.add_row(action["name"], changes_str) verb = "Would update" if dry_run else "Updated" _console.print(f"[bold]{verb} {len(actions)} experiment(s)[/]\n") _console.print(table) if not dry_run: _console.print("\n[green]✓ Migration complete. Index rebuilt.[/]") return 0 def cmd_rebuild_index(args: argparse.Namespace) -> int: lib = _exp_lib() journal = _JOURNAL_PATH if _JOURNAL_PATH.exists() else None index = lib.rebuild_experiment_index(configs_dir=_EXPERIMENTS_DIR, journal_path=journal) count = len(index.get("experiments", {})) _console.print(f"[green]✓ Index rebuilt:[/] {count} experiments indexed at {_EXPERIMENTS_DIR / '.index.json'}") return 0 # --------------------------------------------------------------------------- # CLI wiring # --------------------------------------------------------------------------- def _print_help() -> None: _console.print(Panel( "[bold cyan]fithia2 exp[/] — Experiment Management\n" "[dim]전략 실험 생성, 검색, 계보 추적, 비교[/]", border_style="cyan", padding=(0, 2), )) table = Table(box=box.SIMPLE, show_header=True, header_style="bold yellow", padding=(0, 2)) table.add_column("Command", style="bold green", no_wrap=True) table.add_column("Description") table.add_column("Key Options", style="dim") table.add_row("create", "부모에서 새 실험 생성", "--parent NAME --name NAME [--changelog TEXT] [--created-by TEXT]") table.add_row("search", "실험 검색", "[--tag TAG] [--status STATUS] [--family FAMILY] [--parent NAME] [--pattern REGEX]") table.add_row("tree", "계보 트리 시각화", "ANCESTOR") table.add_row("info", "단일 실험 상세 조회", "NAME") table.add_row("diff", "두 실험 설정 비교", "NAME_A NAME_B") table.add_row("promote", "status → promoted", "NAME [--alias ALIAS]") table.add_row("retire", "status → retired", "NAME") table.add_row("validate", "전체 스키마 검증", "[--fix]") table.add_row("migrate", "기존 파일 메타데이터 백필", "[--dry-run]") table.add_row("rebuild-index", "인덱스 캐시 재생성", "") _console.print(table) def main() -> None: if len(sys.argv) < 2 or sys.argv[1] in ("-h", "--help"): _print_help() return subcmd = sys.argv[1] rest = sys.argv[2:] if subcmd == "create": parser = argparse.ArgumentParser(prog="fithia2 exp create") parser.add_argument("--parent", "-p", required=True, help="Parent experiment name") parser.add_argument("--name", "-n", required=True, help="New experiment name") parser.add_argument("--changelog", "-c", help="Description of changes from parent") parser.add_argument("--created-by", default="ai_agent", help="Creator identifier") args = parser.parse_args(rest) sys.exit(cmd_create(args)) elif subcmd == "search": parser = argparse.ArgumentParser(prog="fithia2 exp search") parser.add_argument("--tag", help="Filter by tag") parser.add_argument("--status", help="Filter by status (draft|active|promoted|retired)") parser.add_argument("--family", help="Filter by version_family (e.g. v6new, v8)") parser.add_argument("--parent", help="Filter by parent experiment name") parser.add_argument("--pattern", help="Filter by name regex pattern") args = parser.parse_args(rest) sys.exit(cmd_search(args)) elif subcmd == "tree": parser = argparse.ArgumentParser(prog="fithia2 exp tree") parser.add_argument("ancestor", help="Root experiment for the tree") args = parser.parse_args(rest) sys.exit(cmd_tree(args)) elif subcmd == "info": parser = argparse.ArgumentParser(prog="fithia2 exp info") parser.add_argument("name", help="Experiment name") args = parser.parse_args(rest) sys.exit(cmd_info(args)) elif subcmd == "diff": parser = argparse.ArgumentParser(prog="fithia2 exp diff") parser.add_argument("name_a", help="First experiment name") parser.add_argument("name_b", help="Second experiment name") args = parser.parse_args(rest) sys.exit(cmd_diff(args)) elif subcmd == "promote": parser = argparse.ArgumentParser(prog="fithia2 exp promote") parser.add_argument("name", help="Experiment name") parser.add_argument("--alias", help="Human-readable alias to add") args = parser.parse_args(rest) sys.exit(cmd_promote(args)) elif subcmd == "retire": parser = argparse.ArgumentParser(prog="fithia2 exp retire") parser.add_argument("name", help="Experiment name") args = parser.parse_args(rest) sys.exit(cmd_retire(args)) elif subcmd == "validate": parser = argparse.ArgumentParser(prog="fithia2 exp validate") parser.add_argument("--fix", action="store_true", help="Auto-fix fixable issues") args = parser.parse_args(rest) sys.exit(cmd_validate(args)) elif subcmd == "migrate": parser = argparse.ArgumentParser(prog="fithia2 exp migrate") parser.add_argument("--dry-run", action="store_true", help="Report changes without writing") args = parser.parse_args(rest) sys.exit(cmd_migrate(args)) elif subcmd in ("rebuild-index", "rebuild_index"): args = argparse.Namespace() sys.exit(cmd_rebuild_index(args)) else: _console.print(f"[red]Unknown subcommand:[/] {subcmd!r}") _print_help() sys.exit(1) if __name__ == "__main__": main()