You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
436 lines
16 KiB
Python
436 lines
16 KiB
Python
"""SQS computation and scoring API endpoints."""
|
|
from __future__ import annotations
|
|
|
|
import threading
|
|
import uuid
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from fastapi import APIRouter, HTTPException
|
|
from pydantic import BaseModel
|
|
|
|
from apps.web.dependencies import get_configs_dir, get_journal_dir, get_runs_dir
|
|
from libs.backtest.tracker import (
|
|
_hydrate_split_result,
|
|
compute_public_sqs_v9,
|
|
compute_rqs,
|
|
compute_wfqs_v2,
|
|
load_journal,
|
|
refresh_public_scores,
|
|
)
|
|
|
|
router = APIRouter(prefix="/sqs", tags=["sqs"])
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# In-memory pipeline state
|
|
# ---------------------------------------------------------------------------
|
|
_pipelines: dict[str, dict[str, Any]] = {}
|
|
_pipelines_lock = threading.Lock()
|
|
|
|
|
|
class RescoreRequest(BaseModel):
|
|
target: str
|
|
|
|
|
|
class PipelineRequest(BaseModel):
|
|
target: str
|
|
|
|
|
|
_NO_ENTRY_RESPONSE = {
|
|
"entry_id": None,
|
|
"sqs_score": None,
|
|
"source": None,
|
|
"pending_reason": "no_backtest_data",
|
|
"pillars": {
|
|
"rqs_score": None, "wfqs_v2_score": None,
|
|
"regime_score": None, "regime_source": "none", "core_score": None,
|
|
},
|
|
"gates": {
|
|
"deployment_gate_factor": None,
|
|
"robustness_gate_factor": None,
|
|
"activity_factor": None,
|
|
},
|
|
"breakdown": {},
|
|
"stored_sqs_score": None,
|
|
"scenario_robustness_score": None,
|
|
"overfit_check_score": None,
|
|
}
|
|
|
|
|
|
def _registry_fallback(journal_dir: Path, experiment_name: str) -> dict[str, Any] | None:
|
|
"""Look up stored SQS from registry when no live journal entry exists."""
|
|
from libs.backtest.domain import ExperimentRegistry
|
|
registry_path = journal_dir / "experiment_registry.json"
|
|
if not registry_path.exists():
|
|
return None
|
|
try:
|
|
registry = ExperimentRegistry.model_validate_json(registry_path.read_text())
|
|
except Exception:
|
|
return None
|
|
|
|
matches = [e for e in registry.entries if e.experiment_name == experiment_name]
|
|
if not matches:
|
|
return None
|
|
entry = sorted(matches, key=lambda e: e.timestamp)[-1]
|
|
|
|
return {
|
|
"entry_id": entry.entry_id,
|
|
"sqs_score": entry.sqs_score,
|
|
"source": "registry_stored",
|
|
"pending_reason": None if entry.sqs_score is not None else "no_backtest_data",
|
|
"pillars": {
|
|
"rqs_score": entry.rqs_score,
|
|
"wfqs_v2_score": entry.wfqs_v2_score,
|
|
"regime_score": None,
|
|
"regime_source": "stored",
|
|
"core_score": None,
|
|
},
|
|
"gates": {
|
|
"deployment_gate_factor": None,
|
|
"robustness_gate_factor": None,
|
|
"activity_factor": None,
|
|
},
|
|
"breakdown": {},
|
|
"stored_sqs_score": entry.sqs_score,
|
|
"scenario_robustness_score": entry.scenario_robustness_score,
|
|
"overfit_check_score": entry.overfit_check_score,
|
|
}
|
|
|
|
|
|
def _compute_for_entry(entry: Any) -> dict[str, Any]:
|
|
"""Run SQS v9 computation on a journal entry and return structured result."""
|
|
train_result = _hydrate_split_result(entry.results.get("train"))
|
|
valid_result = _hydrate_split_result(entry.results.get("valid"))
|
|
test_result = _hydrate_split_result(entry.results.get("test"))
|
|
|
|
rqs_score, _ = compute_rqs(train_result, valid_result, test_result)
|
|
wfqs_v2_score, _ = compute_wfqs_v2(entry.walk_forward_summary)
|
|
|
|
sqs_score, sqs_breakdown, source = compute_public_sqs_v9(
|
|
train_result, valid_result, test_result,
|
|
walk_forward_summary=entry.walk_forward_summary,
|
|
robustness_matrix_summary=entry.robustness_matrix_summary,
|
|
out_of_time_robustness_summary=entry.out_of_time_robustness_summary,
|
|
common_window_summary=entry.common_window_summary,
|
|
reset_common_window_summary=entry.reset_common_window_summary,
|
|
scenario_robustness_score=entry.scenario_robustness_score,
|
|
rqs_score=rqs_score,
|
|
wfqs_v2_score=wfqs_v2_score,
|
|
)
|
|
|
|
regime_src = (
|
|
"scenario_rrs" if entry.scenario_robustness_score is not None
|
|
else "oot_quality" if entry.out_of_time_robustness_summary is not None
|
|
else "neutral_50"
|
|
)
|
|
|
|
pending_reason = None
|
|
if sqs_score is None:
|
|
pending_key = next((k for k in sqs_breakdown if k.startswith("requires_")), None)
|
|
pending_reason = (
|
|
pending_key.replace("requires_", "").replace("_", " ")
|
|
if pending_key else "missing data"
|
|
)
|
|
|
|
return {
|
|
"experiment_name": entry.experiment_name,
|
|
"entry_id": entry.entry_id,
|
|
"sqs_score": sqs_score,
|
|
"source": source,
|
|
"pending_reason": pending_reason,
|
|
"pillars": {
|
|
"rqs_score": rqs_score,
|
|
"wfqs_v2_score": wfqs_v2_score,
|
|
"regime_score": sqs_breakdown.get("regime_score"),
|
|
"regime_source": regime_src,
|
|
"core_score": sqs_breakdown.get("core_score"),
|
|
},
|
|
"gates": {
|
|
"deployment_gate_factor": sqs_breakdown.get("deployment_gate_factor"),
|
|
"robustness_gate_factor": sqs_breakdown.get("gate_factor"),
|
|
"activity_factor": sqs_breakdown.get("activity_factor"),
|
|
},
|
|
"breakdown": sqs_breakdown,
|
|
"stored_sqs_score": entry.sqs_score,
|
|
"scenario_robustness_score": entry.scenario_robustness_score,
|
|
"overfit_check_score": entry.overfit_check_score,
|
|
}
|
|
|
|
|
|
@router.get("/{experiment_name:path}")
|
|
def get_sqs(experiment_name: str) -> dict[str, Any]:
|
|
"""Compute SQS v9 live for an experiment (read-only, no journal write).
|
|
|
|
Falls back to registry-stored SQS when no journal entry exists.
|
|
Never returns 404 — always returns a structured response.
|
|
"""
|
|
journal_dir = get_journal_dir()
|
|
journal_path = journal_dir / "improvement_journal.jsonl"
|
|
|
|
base = {"experiment_name": experiment_name}
|
|
|
|
# 1. Try journal — live recompute
|
|
if journal_path.exists():
|
|
entries = load_journal(journal_path)
|
|
exact = [e for e in entries if e.experiment_name == experiment_name]
|
|
if exact:
|
|
entry = sorted(exact, key=lambda e: e.timestamp)[-1]
|
|
return {**base, **_compute_for_entry(entry)}
|
|
|
|
# 2. Fall back to registry (stored values, no live recompute)
|
|
reg = _registry_fallback(journal_dir, experiment_name)
|
|
if reg:
|
|
return {**base, **reg}
|
|
|
|
# 3. Nothing available
|
|
return {**base, **_NO_ENTRY_RESPONSE}
|
|
|
|
|
|
def _run_pipeline_thread(pipeline_id: str, experiment_name: str) -> None:
|
|
"""Background thread: run train/valid/test backtests → record → compute SQS."""
|
|
import datetime as _dt
|
|
|
|
from libs.backtest.tracker import (
|
|
append_journal_entry,
|
|
build_split_result,
|
|
get_next_entry_id,
|
|
journal_lock,
|
|
rebuild_registry,
|
|
scan_runs_for_experiment,
|
|
)
|
|
from libs.backtest.domain import JournalEntry
|
|
|
|
def _set(updates: dict) -> None:
|
|
with _pipelines_lock:
|
|
_pipelines[pipeline_id].update(updates)
|
|
|
|
try:
|
|
configs_dir = get_configs_dir()
|
|
journal_dir = get_journal_dir()
|
|
runs_dir = get_runs_dir()
|
|
journal_path = journal_dir / "improvement_journal.jsonl"
|
|
registry_path = journal_dir / "experiment_registry.json"
|
|
leaderboard_path = journal_dir / "LEADERBOARD.md"
|
|
config_path = configs_dir / f"{experiment_name}.json"
|
|
|
|
if not config_path.exists():
|
|
_set({"status": "failed", "error": f"Config not found: {experiment_name}.json"})
|
|
return
|
|
|
|
# Import backtester internals
|
|
from apps.backtester.run import (
|
|
BacktestRunner,
|
|
_build_merged_snapshot_store,
|
|
_extend_store_to_requested_window,
|
|
load_manifest,
|
|
resolve_config,
|
|
)
|
|
from libs.backtest.snapshots import resolve_snapshot_path
|
|
import pyarrow.parquet as _pq
|
|
|
|
_set({"status": "running", "step": "Loading snapshot store…"})
|
|
manifest = load_manifest(str(config_path))
|
|
config = resolve_config(manifest)
|
|
|
|
# Build merged snapshot store — uses cached PKL, completes in seconds
|
|
store = _build_merged_snapshot_store(manifest, config, snapshot_dir_override=None)
|
|
|
|
# Determine split date ranges from parquet files (read only execution_date column)
|
|
snapshot_request_id = config.requested_snapshot_id or config.dataset_snapshot_id
|
|
snapshot_dir = resolve_snapshot_path(snapshot_request_id)
|
|
if snapshot_dir is None:
|
|
_set({"status": "failed", "error": f"Snapshot directory not found for '{snapshot_request_id}'"})
|
|
return
|
|
|
|
split_date_ranges: dict[str, tuple[_dt.date, _dt.date]] = {}
|
|
for split in ("train", "valid", "test"):
|
|
parquet_path = snapshot_dir / f"{split}.parquet"
|
|
if not parquet_path.exists():
|
|
_set({"status": "failed", "error": f"Parquet not found: {parquet_path}"})
|
|
return
|
|
tbl = _pq.read_table(str(parquet_path), columns=["execution_date"])
|
|
raw_dates = tbl["execution_date"].to_pylist()
|
|
dates: list[_dt.date] = []
|
|
for v in raw_dates:
|
|
if v is None:
|
|
continue
|
|
if isinstance(v, _dt.datetime):
|
|
dates.append(v.date())
|
|
elif isinstance(v, _dt.date):
|
|
dates.append(v)
|
|
elif isinstance(v, str):
|
|
dates.append(_dt.date.fromisoformat(v[:10]))
|
|
if not dates:
|
|
_set({"status": "failed", "error": f"No execution dates in {split}.parquet"})
|
|
return
|
|
split_date_ranges[split] = (min(dates), max(dates))
|
|
|
|
# Run each split in-process using the cached merged store (slice per split)
|
|
for split in ("train", "valid", "test"):
|
|
_set({"status": "running", "step": f"Running {split} split…"})
|
|
split_start, split_end = split_date_ranges[split]
|
|
split_store = store.slice_by_date_range(split_start, split_end)
|
|
split_store = _extend_store_to_requested_window(
|
|
store=split_store,
|
|
config=config,
|
|
start_date=split_start,
|
|
end_date=split_end,
|
|
snapshot_dir_override=None,
|
|
)
|
|
runner = BacktestRunner(
|
|
manifest=manifest,
|
|
config=config,
|
|
store=split_store,
|
|
initial_equity=10_000.0,
|
|
split_name=split,
|
|
)
|
|
runner.run(output_root=str(runs_dir))
|
|
|
|
# Record
|
|
_set({"status": "running", "step": "Recording results to journal…"})
|
|
split_runs = scan_runs_for_experiment(runs_dir, experiment_name)
|
|
if not split_runs:
|
|
_set({"status": "failed", "error": "Backtests completed but no runs found in runs/. Check output-root."})
|
|
return
|
|
|
|
results: dict[str, Any] = {}
|
|
for sname, (run_id, metrics) in split_runs.items():
|
|
results[sname] = build_split_result(sname, run_id, metrics)
|
|
|
|
rqs_score, rqs_breakdown = compute_rqs(results.get("train"), results.get("valid"), results.get("test"))
|
|
sqs_score, sqs_breakdown, _ = compute_public_sqs_v9(
|
|
results.get("train"), results.get("valid"), results.get("test"),
|
|
rqs_score=rqs_score, wfqs_v2_score=None,
|
|
)
|
|
tags = [t for t in experiment_name.replace("-", "_").split("_") if t]
|
|
entry_id = get_next_entry_id(journal_path)
|
|
timestamp = _dt.datetime.now(_dt.timezone.utc).isoformat()
|
|
entry = JournalEntry(
|
|
entry_id=entry_id,
|
|
timestamp=timestamp,
|
|
experiment_name=experiment_name,
|
|
hypothesis="Auto-recorded via web GUI",
|
|
results=results,
|
|
rqs_score=rqs_score,
|
|
rqs_breakdown=rqs_breakdown,
|
|
sqs_score=sqs_score,
|
|
sqs_breakdown=sqs_breakdown,
|
|
verdict="unknown",
|
|
tags=tags,
|
|
)
|
|
with journal_lock(journal_path):
|
|
append_journal_entry(journal_path, entry)
|
|
rebuild_registry(journal_path, registry_path, leaderboard_path)
|
|
|
|
# Compute final SQS
|
|
_set({"status": "running", "step": "Computing SQS v9…"})
|
|
fresh_entries = load_journal(journal_path)
|
|
fresh_exact = [e for e in fresh_entries if e.experiment_name == experiment_name]
|
|
fresh_entry = sorted(fresh_exact, key=lambda e: e.timestamp)[-1]
|
|
sqs_result = _compute_for_entry(fresh_entry)
|
|
|
|
_set({"status": "completed", "step": "Done", "result": sqs_result})
|
|
|
|
except Exception as exc:
|
|
_set({"status": "failed", "error": str(exc)})
|
|
|
|
|
|
@router.post("/pipeline")
|
|
def pipeline_start(req: PipelineRequest) -> dict[str, Any]:
|
|
"""Start the auto-backtest pipeline: run train/valid/test → record → compute SQS.
|
|
|
|
Returns a pipeline_id to poll with GET /sqs/pipeline/{pipeline_id}.
|
|
If a journal entry already exists, computes SQS immediately (no pipeline).
|
|
"""
|
|
experiment_name = req.target
|
|
journal_dir = get_journal_dir()
|
|
journal_path = journal_dir / "improvement_journal.jsonl"
|
|
base = {"experiment_name": experiment_name}
|
|
|
|
# Journal entry already exists → compute immediately
|
|
if journal_path.exists():
|
|
entries = load_journal(journal_path)
|
|
exact = [e for e in entries if e.experiment_name == experiment_name]
|
|
if exact:
|
|
entry = sorted(exact, key=lambda e: e.timestamp)[-1]
|
|
return {**base, "pipeline_status": "existing_entry", **_compute_for_entry(entry)}
|
|
|
|
# Config must exist before we start
|
|
config_path = get_configs_dir() / f"{experiment_name}.json"
|
|
if not config_path.exists():
|
|
raise HTTPException(status_code=404, detail=f"Experiment config not found: {experiment_name}")
|
|
|
|
pipeline_id = str(uuid.uuid4())[:8]
|
|
with _pipelines_lock:
|
|
_pipelines[pipeline_id] = {
|
|
"pipeline_id": pipeline_id,
|
|
"experiment_name": experiment_name,
|
|
"status": "running",
|
|
"step": "Starting…",
|
|
"error": None,
|
|
"result": None,
|
|
}
|
|
|
|
thread = threading.Thread(
|
|
target=_run_pipeline_thread,
|
|
args=(pipeline_id, experiment_name),
|
|
daemon=True,
|
|
)
|
|
thread.start()
|
|
return {**base, "pipeline_id": pipeline_id, "pipeline_status": "running", "step": "Starting…"}
|
|
|
|
|
|
@router.get("/pipeline/{pipeline_id}")
|
|
def pipeline_poll(pipeline_id: str) -> dict[str, Any]:
|
|
"""Poll pipeline status. Returns result when status='completed'."""
|
|
with _pipelines_lock:
|
|
state = _pipelines.get(pipeline_id)
|
|
if state is None:
|
|
raise HTTPException(status_code=404, detail=f"Pipeline not found: {pipeline_id}")
|
|
out = dict(state)
|
|
if out.get("result"):
|
|
out.update(out.pop("result"))
|
|
out["pipeline_status"] = out.pop("status")
|
|
return out
|
|
|
|
|
|
@router.post("/rescore")
|
|
def rescore_experiment(req: RescoreRequest) -> dict[str, Any]:
|
|
"""Recompute SQS for one experiment, save to journal, rebuild leaderboard."""
|
|
from libs.backtest.tracker import journal_lock, rebuild_registry
|
|
|
|
journal_dir = get_journal_dir()
|
|
journal_path = journal_dir / "improvement_journal.jsonl"
|
|
registry_path = journal_dir / "experiment_registry.json"
|
|
leaderboard_path = journal_dir / "LEADERBOARD.md"
|
|
|
|
if not journal_path.exists():
|
|
raise HTTPException(status_code=404, detail="Journal not found")
|
|
|
|
entries = load_journal(journal_path)
|
|
exact = [e for e in entries if e.experiment_name == req.target]
|
|
if not exact:
|
|
raise HTTPException(
|
|
status_code=404,
|
|
detail=f"No journal entry for '{req.target}'. Run a backtest and record results first (fithia2 rec).",
|
|
)
|
|
entry = sorted(exact, key=lambda e: e.timestamp)[-1]
|
|
target_name = entry.experiment_name
|
|
|
|
def _only_this(e: Any) -> bool:
|
|
return e.experiment_name == target_name
|
|
|
|
with journal_lock(journal_path):
|
|
updated_count = refresh_public_scores(journal_path, selector=_only_this)
|
|
rebuild_registry(journal_path, registry_path, leaderboard_path)
|
|
|
|
fresh_entries = load_journal(journal_path)
|
|
fresh_exact = [e for e in fresh_entries if e.experiment_name == target_name]
|
|
fresh_entry = sorted(fresh_exact, key=lambda e: e.timestamp)[-1]
|
|
result = {
|
|
"experiment_name": target_name,
|
|
**_compute_for_entry(fresh_entry),
|
|
"entries_updated": updated_count,
|
|
}
|
|
return result
|