fithia2/apps/web/routers/leaderboard.py

"""Leaderboard API endpoints."""
from __future__ import annotations

import json
from pathlib import Path
from typing import Any

from fastapi import APIRouter, HTTPException, Query

from apps.web.dependencies import get_configs_dir, get_journal_dir
from libs.backtest.domain import ExperimentRegistry, RegistryEntry
from libs.backtest.experiments import _load_index as load_experiment_index
from libs.backtest.tracker import filter_registry_entries

router = APIRouter(prefix="/leaderboard", tags=["leaderboard"])

def _sqs_sort_key(e: Any) -> tuple:  # type: ignore[return]
    """SQS primary, then visible return metrics as tiebreakers."""
    return (
        e.sqs_score is None,
        -(e.sqs_score or 0),
        # Tiebreakers: visible table metrics (higher is better, except DD)
        e.valid_total_return_pct is None,
        -(e.valid_total_return_pct or 0),
        e.total_return_pct is None,
        -(e.total_return_pct or 0),
        e.annualized_return_pct is None,
        -(e.annualized_return_pct or 0),
        e.max_drawdown_pct is None,
        +(e.max_drawdown_pct or 100.0),  # lower DD is better
    )


_SORT_KEYS = {
    "sqs": _sqs_sort_key,
    "rqs": lambda e: (e.rqs_score is None, -(e.rqs_score or 0)),
    "wfqs": lambda e: (e.wfqs_score is None, -(e.wfqs_score or 0)),
    "deployment": lambda e: (e.deployment_score is None, -(e.deployment_score or 0)),
    "promotion": lambda e: (e.promotion_score is None, -(e.promotion_score or 0)),
    "unified": lambda e: (e.unified_score is None, -(e.unified_score or 0)),
    "return": lambda e: (e.total_return_pct is None, -(e.total_return_pct or 0)),
    "valid_return": lambda e: (e.valid_total_return_pct is None, -(e.valid_total_return_pct or 0)),
}


def _load_registry(journal_dir: Path) -> ExperimentRegistry:
    registry_path = journal_dir / "experiment_registry.json"
    if not registry_path.exists():
        return ExperimentRegistry()
    return ExperimentRegistry.model_validate_json(registry_path.read_text())


def _entry_to_dict(entry: RegistryEntry, rank: int, exp_id_map: dict[str, int | None] | None = None) -> dict[str, Any]:
    d = entry.model_dump()
    d["rank"] = rank
    d["config_id"] = exp_id_map.get(entry.experiment_name) if exp_id_map else None
    # Drop heavy nested objects from list view to keep payload small
    d.pop("walk_forward_summary", None)
    d.pop("robustness_matrix_summary", None)
    d.pop("out_of_time_robustness_summary", None)
    d.pop("common_window_summary", None)
    d.pop("reset_common_window_summary", None)
    d.pop("multi_capital_common_window_summary", None)
    return d


@router.get("")
def get_leaderboard(
    sort_by: str = Query("sqs", description="sqs | rqs | wfqs | deployment | promotion | unified | return"),
    include_retired: bool = Query(False),
    top: int = Query(100, le=500),
    offset: int = 0,
    family: str | None = Query(None),
) -> dict[str, Any]:
    """Get the ranked leaderboard."""
    journal_dir = get_journal_dir()
    registry = _load_registry(journal_dir)

    # Build experiment id map from index
    try:
        index = load_experiment_index(get_configs_dir())
        exp_id_map: dict[str, int | None] = {name: meta.get("id") for name, meta in index.get("experiments", {}).items()}
    except Exception:
        exp_id_map = {}

    filtered = filter_registry_entries(registry.entries, include_retired=include_retired)

    # Also exclude experiments with status="retired" in their config (unless include_retired)
    if not include_retired:
        try:
            index = load_experiment_index(get_configs_dir())
            retired_in_config = {
                name for name, meta in index.get("experiments", {}).items()
                if meta.get("status") == "retired"
            }
            filtered = [e for e in filtered if e.experiment_name not in retired_in_config]
        except Exception:
            pass

    if family:
        from libs.backtest.tracker import classify_strategy_family
        filtered = [e for e in filtered if classify_strategy_family(e.experiment_name, []) == family]

    sort_fn = _SORT_KEYS.get(sort_by, _SORT_KEYS["sqs"])
    ranked = sorted(filtered, key=sort_fn)

    total = len(ranked)
    page = ranked[offset : offset + top]
    entries = [_entry_to_dict(e, i + offset + 1, exp_id_map) for i, e in enumerate(page)]

    return {"entries": entries, "total": total, "sort_by": sort_by}


@router.get("/entry/{entry_id}")
def get_entry(entry_id: str) -> dict[str, Any]:
    """Get full journal entry detail including walk-forward and robustness summaries."""
    journal_dir = get_journal_dir()
    registry = _load_registry(journal_dir)

    for entry in registry.entries:
        if entry.entry_id == entry_id:
            return entry.model_dump()

    raise HTTPException(status_code=404, detail=f"Entry not found: {entry_id}")


@router.get("/experiment/{name}")
def get_experiment_entry(name: str) -> dict[str, Any]:
    """Get registry entry for a specific experiment name."""
    journal_dir = get_journal_dir()
    registry = _load_registry(journal_dir)

    # Return the most recent entry for this experiment
    matches = [e for e in registry.entries if e.experiment_name == name]
    if not matches:
        raise HTTPException(status_code=404, detail=f"No registry entry for: {name}")

    # Sort by timestamp descending; return the most recent
    matches.sort(key=lambda e: e.timestamp, reverse=True)
    return matches[0].model_dump()


@router.post("/refresh")
def refresh_leaderboard() -> dict[str, Any]:
    """Trigger a full registry + leaderboard rebuild from journal."""
    from libs.backtest.tracker import rebuild_registry

    journal_dir = get_journal_dir()
    journal_path = journal_dir / "improvement_journal.jsonl"
    registry_path = journal_dir / "experiment_registry.json"
    leaderboard_path = journal_dir / "LEADERBOARD.md"

    if not journal_path.exists():
        raise HTTPException(status_code=404, detail="Journal file not found")

    registry = rebuild_registry(journal_path, registry_path, leaderboard_path)
    return {"entries": len(registry.entries), "rebuilt": True}