"""Leaderboard API endpoints.""" from __future__ import annotations import json from pathlib import Path from typing import Any from fastapi import APIRouter, HTTPException, Query from apps.web.dependencies import get_configs_dir, get_journal_dir from libs.backtest.domain import ExperimentRegistry, RegistryEntry from libs.backtest.experiments import _load_index as load_experiment_index from libs.backtest.tracker import filter_registry_entries router = APIRouter(prefix="/leaderboard", tags=["leaderboard"]) def _sqs_sort_key(e: Any) -> tuple: # type: ignore[return] """SQS primary, then visible return metrics as tiebreakers.""" return ( e.sqs_score is None, -(e.sqs_score or 0), # Tiebreakers: visible table metrics (higher is better, except DD) e.valid_total_return_pct is None, -(e.valid_total_return_pct or 0), e.total_return_pct is None, -(e.total_return_pct or 0), e.annualized_return_pct is None, -(e.annualized_return_pct or 0), e.max_drawdown_pct is None, +(e.max_drawdown_pct or 100.0), # lower DD is better ) _SORT_KEYS = { "sqs": _sqs_sort_key, "rqs": lambda e: (e.rqs_score is None, -(e.rqs_score or 0)), "wfqs": lambda e: (e.wfqs_score is None, -(e.wfqs_score or 0)), "deployment": lambda e: (e.deployment_score is None, -(e.deployment_score or 0)), "promotion": lambda e: (e.promotion_score is None, -(e.promotion_score or 0)), "unified": lambda e: (e.unified_score is None, -(e.unified_score or 0)), "return": lambda e: (e.total_return_pct is None, -(e.total_return_pct or 0)), "valid_return": lambda e: (e.valid_total_return_pct is None, -(e.valid_total_return_pct or 0)), } def _load_registry(journal_dir: Path) -> ExperimentRegistry: registry_path = journal_dir / "experiment_registry.json" if not registry_path.exists(): return ExperimentRegistry() return ExperimentRegistry.model_validate_json(registry_path.read_text()) def _entry_to_dict(entry: RegistryEntry, rank: int, exp_id_map: dict[str, int | None] | None = None) -> dict[str, Any]: d = entry.model_dump() d["rank"] = rank d["config_id"] = exp_id_map.get(entry.experiment_name) if exp_id_map else None # Drop heavy nested objects from list view to keep payload small d.pop("walk_forward_summary", None) d.pop("robustness_matrix_summary", None) d.pop("out_of_time_robustness_summary", None) d.pop("common_window_summary", None) d.pop("reset_common_window_summary", None) d.pop("multi_capital_common_window_summary", None) return d @router.get("") def get_leaderboard( sort_by: str = Query("sqs", description="sqs | rqs | wfqs | deployment | promotion | unified | return"), include_retired: bool = Query(False), top: int = Query(100, le=500), offset: int = 0, family: str | None = Query(None), ) -> dict[str, Any]: """Get the ranked leaderboard.""" journal_dir = get_journal_dir() registry = _load_registry(journal_dir) # Build experiment id map from index try: index = load_experiment_index(get_configs_dir()) exp_id_map: dict[str, int | None] = {name: meta.get("id") for name, meta in index.get("experiments", {}).items()} except Exception: exp_id_map = {} filtered = filter_registry_entries(registry.entries, include_retired=include_retired) # Also exclude experiments with status="retired" in their config (unless include_retired) if not include_retired: try: index = load_experiment_index(get_configs_dir()) retired_in_config = { name for name, meta in index.get("experiments", {}).items() if meta.get("status") == "retired" } filtered = [e for e in filtered if e.experiment_name not in retired_in_config] except Exception: pass if family: from libs.backtest.tracker import classify_strategy_family filtered = [e for e in filtered if classify_strategy_family(e.experiment_name, []) == family] sort_fn = _SORT_KEYS.get(sort_by, _SORT_KEYS["sqs"]) ranked = sorted(filtered, key=sort_fn) total = len(ranked) page = ranked[offset : offset + top] entries = [_entry_to_dict(e, i + offset + 1, exp_id_map) for i, e in enumerate(page)] return {"entries": entries, "total": total, "sort_by": sort_by} @router.get("/entry/{entry_id}") def get_entry(entry_id: str) -> dict[str, Any]: """Get full journal entry detail including walk-forward and robustness summaries.""" journal_dir = get_journal_dir() registry = _load_registry(journal_dir) for entry in registry.entries: if entry.entry_id == entry_id: return entry.model_dump() raise HTTPException(status_code=404, detail=f"Entry not found: {entry_id}") @router.get("/experiment/{name}") def get_experiment_entry(name: str) -> dict[str, Any]: """Get registry entry for a specific experiment name.""" journal_dir = get_journal_dir() registry = _load_registry(journal_dir) # Return the most recent entry for this experiment matches = [e for e in registry.entries if e.experiment_name == name] if not matches: raise HTTPException(status_code=404, detail=f"No registry entry for: {name}") # Sort by timestamp descending; return the most recent matches.sort(key=lambda e: e.timestamp, reverse=True) return matches[0].model_dump() @router.post("/refresh") def refresh_leaderboard() -> dict[str, Any]: """Trigger a full registry + leaderboard rebuild from journal.""" from libs.backtest.tracker import rebuild_registry journal_dir = get_journal_dir() journal_path = journal_dir / "improvement_journal.jsonl" registry_path = journal_dir / "experiment_registry.json" leaderboard_path = journal_dir / "LEADERBOARD.md" if not journal_path.exists(): raise HTTPException(status_code=404, detail="Journal file not found") registry = rebuild_registry(journal_path, registry_path, leaderboard_path) return {"entries": len(registry.entries), "rebuilt": True}