You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
330 lines
13 KiB
Python
330 lines
13 KiB
Python
"""ORB scenario test with streaming intraday fetch."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from apps.intraday_bt.oracle import make_intraday_oracle_client
|
|
from rich import box
|
|
from rich.console import Console
|
|
from rich.panel import Panel
|
|
from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
|
|
from rich.table import Table
|
|
|
|
from libs.common.config import get_settings
|
|
from libs.intraday.domain import ORBStrategyParams
|
|
from libs.oracle_client import OracleClient
|
|
|
|
from apps.intraday_bt.orb_research import (
|
|
build_orb_research_context,
|
|
compute_orb_rrs,
|
|
filter_days,
|
|
force_simple_returns,
|
|
resolve_orb_config,
|
|
simulate_orb_period,
|
|
)
|
|
from apps.intraday_bt.run import _latest_backtest_date
|
|
|
|
_console = Console(width=120)
|
|
_DEFAULT_START_DATE = "2022-01-01"
|
|
|
|
|
|
SCENARIO_REGISTRY: dict[str, dict[str, Any]] = {
|
|
"bear_2022": {
|
|
"description": "2022 bear market — fed hikes, tech selloff",
|
|
"start": "2022-01-03",
|
|
"end": "2022-12-30",
|
|
"group": "regime",
|
|
"expected": "negative (long-only headwind)",
|
|
},
|
|
"recovery_2023h1": {
|
|
"description": "Early 2023 recovery from bear market lows",
|
|
"start": "2023-01-03",
|
|
"end": "2023-06-30",
|
|
"group": "regime",
|
|
"expected": "positive (volatility + momentum)",
|
|
},
|
|
"bull_2023h2": {
|
|
"description": "Strong H2 2023 AI-driven bull run",
|
|
"start": "2023-07-03",
|
|
"end": "2023-12-29",
|
|
"group": "regime",
|
|
"expected": "positive (strong trend)",
|
|
},
|
|
"mixed_2024": {
|
|
"description": "Mixed 2024 — rate-cut expectations, choppy mid-year",
|
|
"start": "2024-01-02",
|
|
"end": "2024-12-31",
|
|
"group": "regime",
|
|
"expected": "moderate",
|
|
},
|
|
"bull_2025": {
|
|
"description": "2025 continuation bull market",
|
|
"start": "2025-01-02",
|
|
"end": "2025-12-31",
|
|
"group": "regime",
|
|
"expected": "positive",
|
|
},
|
|
"oos_2026": {
|
|
"description": "Pure OOS holdout — 2026 YTD (never seen in IS)",
|
|
"start": "2026-01-02",
|
|
"end": None,
|
|
"group": "regime",
|
|
"expected": "validation only",
|
|
},
|
|
"no_rvol_filter": {
|
|
"description": "Full period, RVOL filter disabled (min_rvol=0)",
|
|
"start": None,
|
|
"end": None,
|
|
"group": "signal",
|
|
"expected": "should degrade if RVOL adds value",
|
|
"param_override": {"min_rvol": 0.0},
|
|
},
|
|
"random_ranking": {
|
|
"description": "Full period, candidates ranked randomly",
|
|
"start": None,
|
|
"end": None,
|
|
"group": "signal",
|
|
"expected": "should degrade if ranking signal is real",
|
|
"shuffle_candidates": True,
|
|
},
|
|
}
|
|
|
|
SCENARIO_GROUPS: dict[str, list[str]] = {
|
|
"regime": ["bear_2022", "recovery_2023h1", "bull_2023h2", "mixed_2024", "bull_2025", "oos_2026"],
|
|
"signal": ["no_rvol_filter", "random_ranking"],
|
|
"quick": ["bear_2022", "bull_2023h2", "oos_2026"],
|
|
"all": list(SCENARIO_REGISTRY.keys()),
|
|
}
|
|
|
|
|
|
async def run_scenario(
|
|
scenario_name: str,
|
|
scenario_def: dict[str, Any],
|
|
context,
|
|
client: OracleClient,
|
|
full_start: str,
|
|
progress_prefix: str = "",
|
|
) -> dict[str, Any]:
|
|
base_params = context.config.orb_strategy or ORBStrategyParams()
|
|
if scenario_def.get("param_override"):
|
|
base_params = base_params.model_copy(update=scenario_def["param_override"])
|
|
|
|
sc_start = scenario_def.get("start") or full_start
|
|
sc_end = scenario_def.get("end") or context.trading_days[-1]
|
|
days = filter_days(context.trading_days, sc_start, sc_end)
|
|
if len(days) < 10:
|
|
return {"scenario": scenario_name, "verdict": "SKIP", "notes": f"Only {len(days)} days in range"}
|
|
|
|
if progress_prefix:
|
|
print(f"{progress_prefix}{scenario_name}: {days[0]} → {days[-1]} ({len(days)} days)")
|
|
|
|
metrics = await simulate_orb_period(
|
|
context,
|
|
client,
|
|
base_params,
|
|
days,
|
|
run_id=f"sc_{scenario_name[:8]}",
|
|
shuffle_candidates_seed=1234 if scenario_def.get("shuffle_candidates") else None,
|
|
progress_prefix=f"{progress_prefix}[{scenario_name}] " if progress_prefix else "",
|
|
)
|
|
return {
|
|
"scenario": scenario_name,
|
|
"period": f"{days[0]} → {days[-1]} ({len(days)} days)",
|
|
"sharpe_ratio": metrics.sharpe_ratio or 0.0,
|
|
"total_return_pct": (metrics.total_return_pct or 0.0) * 100,
|
|
"max_drawdown_pct": abs((metrics.max_drawdown_pct or 0.0) * 100),
|
|
"win_rate": (metrics.win_rate or 0.0) * 100,
|
|
"profit_factor": metrics.profit_factor or 0.0,
|
|
"total_trades": metrics.total_trades or 0,
|
|
}
|
|
|
|
|
|
def _rrs_verdict(rrs: float) -> str:
|
|
if rrs >= 70:
|
|
return "ROBUST"
|
|
if rrs >= 40:
|
|
return "FRAGILE"
|
|
return "OVERFIT"
|
|
|
|
|
|
def _print_scenario_table(scenario_results: dict[str, dict], config_name: str) -> None:
|
|
table = Table(title=f"ORB Scenario Results — {config_name}", box=box.ROUNDED, width=118)
|
|
table.add_column("Scenario", style="cyan", min_width=20)
|
|
table.add_column("Period", style="dim", min_width=26)
|
|
table.add_column("Sharpe", justify="right", min_width=7)
|
|
table.add_column("Return%", justify="right", min_width=9)
|
|
table.add_column("MaxDD%", justify="right", min_width=8)
|
|
table.add_column("Win%", justify="right", min_width=6)
|
|
table.add_column("PF", justify="right", min_width=6)
|
|
table.add_column("Trades", justify="right", min_width=7)
|
|
for name, result in scenario_results.items():
|
|
if result.get("verdict") == "SKIP":
|
|
table.add_row(name, "[dim]SKIP[/dim]", "-", "-", "-", "-", "-", "-")
|
|
continue
|
|
sr = result.get("sharpe_ratio", 0.0)
|
|
ret = result.get("total_return_pct", 0.0)
|
|
dd = result.get("max_drawdown_pct", 0.0)
|
|
win = result.get("win_rate", 0.0)
|
|
pf = result.get("profit_factor", 0.0)
|
|
trades = result.get("total_trades", 0)
|
|
period = result.get("period", "")
|
|
sharpe_str = f"[green]{sr:.2f}[/green]" if sr >= 1.5 else f"[yellow]{sr:.2f}[/yellow]" if sr >= 0.5 else f"[red]{sr:.2f}[/red]" if sr < 0 else f"[dim]{sr:.2f}[/dim]"
|
|
ret_str = f"[green]+{ret:.1f}%[/green]" if ret > 0 else f"[red]{ret:.1f}%[/red]"
|
|
dd_str = f"[red]{dd:.1f}%[/red]" if dd > 15 else f"{dd:.1f}%"
|
|
table.add_row(name, period, sharpe_str, ret_str, dd_str, f"{win:.0f}%", f"{pf:.2f}", str(trades))
|
|
_console.print()
|
|
_console.print(table)
|
|
|
|
|
|
def _score_bar(score: float) -> str:
|
|
filled = int(round(score / 5))
|
|
bar = "█" * filled + "░" * (20 - filled)
|
|
color = "green" if score >= 70 else "yellow" if score >= 40 else "red"
|
|
return f"[{color}]{bar}[/{color}] {score:.0f}/100"
|
|
|
|
|
|
def _print_rrs_panel(rrs: float, components: dict[str, float], config_name: str) -> None:
|
|
verdict = _rrs_verdict(rrs)
|
|
color = "green" if verdict == "ROBUST" else "yellow" if verdict == "FRAGILE" else "red"
|
|
lines = [
|
|
"[bold]ORB REGIME ROBUSTNESS SCORE (RRS)[/bold]",
|
|
f"Strategy: [cyan]{config_name}[/cyan]",
|
|
"",
|
|
f" Bear Survival {_score_bar(components['bear_survival'])}",
|
|
f" Breadth {_score_bar(components['breadth'])}",
|
|
f" Drawdown Resilience {_score_bar(components['drawdown_resilience'])}",
|
|
f" OOS Integrity {_score_bar(components['oos_integrity'])}",
|
|
f" Stability {_score_bar(components['stability'])}",
|
|
"",
|
|
f" [bold]RRS: {_score_bar(rrs)}[/bold]",
|
|
f" [{color} bold]Verdict: {verdict}[/{color} bold]",
|
|
]
|
|
_console.print()
|
|
_console.print(Panel("\n".join(lines), box=box.DOUBLE, width=100))
|
|
|
|
|
|
async def _async_main(args: argparse.Namespace) -> int:
|
|
if args.list or args.config is None:
|
|
_console.print("\n[bold]Available scenarios:[/bold]")
|
|
for name, scenario in SCENARIO_REGISTRY.items():
|
|
start_str = scenario.get("start") or "full period"
|
|
end_str = scenario.get("end") or "present"
|
|
_console.print(f" [cyan]{name:<22}[/cyan] {start_str} → {end_str} {scenario['description']}")
|
|
_console.print("\n[bold]Scenario groups:[/bold]")
|
|
for group, names in SCENARIO_GROUPS.items():
|
|
_console.print(f" [yellow]{group:<10}[/yellow] {', '.join(names)}")
|
|
return 0
|
|
|
|
if args.scenario:
|
|
if args.scenario not in SCENARIO_REGISTRY:
|
|
_console.print(f"[red]Unknown scenario '{args.scenario}'[/red]")
|
|
return 1
|
|
scenario_names = [args.scenario]
|
|
elif args.quick:
|
|
scenario_names = SCENARIO_GROUPS["quick"]
|
|
elif args.group:
|
|
if args.group not in SCENARIO_GROUPS:
|
|
_console.print(f"[red]Unknown group '{args.group}'[/red]")
|
|
return 1
|
|
scenario_names = SCENARIO_GROUPS[args.group]
|
|
else:
|
|
scenario_names = SCENARIO_GROUPS["all"]
|
|
|
|
config_path, config = resolve_orb_config(args.config)
|
|
config = force_simple_returns(config)
|
|
config_slug = config_path.stem
|
|
|
|
_console.print()
|
|
_console.print(
|
|
Panel(
|
|
f"[bold]ORB SCENARIO TEST[/bold]\n"
|
|
f"Config: [cyan]{config_path}[/cyan]\n"
|
|
f"Scenarios: [yellow]{len(scenario_names)}[/yellow] ({', '.join(scenario_names)})\n"
|
|
f"Full data window: {args.start} → {args.end or _latest_backtest_date().isoformat()}",
|
|
box=box.DOUBLE,
|
|
width=100,
|
|
)
|
|
)
|
|
|
|
settings = get_settings()
|
|
async with make_intraday_oracle_client(settings) as client:
|
|
context = await build_orb_research_context(
|
|
config,
|
|
args.start,
|
|
args.end or _latest_backtest_date().isoformat(),
|
|
client,
|
|
print_progress=True,
|
|
)
|
|
|
|
t0 = time.time()
|
|
scenario_results: dict[str, dict] = {}
|
|
with Progress(
|
|
SpinnerColumn(),
|
|
TextColumn("[progress.description]{task.description}"),
|
|
BarColumn(bar_width=30),
|
|
"{task.completed}/{task.total}",
|
|
TimeElapsedColumn(),
|
|
console=_console,
|
|
) as progress:
|
|
task = progress.add_task("Running scenarios...", total=len(scenario_names))
|
|
for name in scenario_names:
|
|
progress.update(task, description=f"[cyan]{name}[/cyan]")
|
|
scenario_results[name] = await run_scenario(name, SCENARIO_REGISTRY[name], context, client, args.start)
|
|
progress.advance(task)
|
|
progress.update(task, description="Complete")
|
|
elapsed = time.time() - t0
|
|
|
|
_print_scenario_table(scenario_results, config_slug)
|
|
non_skipped = {k: v for k, v in scenario_results.items() if v.get("verdict") != "SKIP" and "sharpe_ratio" in v}
|
|
if len(non_skipped) >= 3:
|
|
rrs, components = compute_orb_rrs(non_skipped)
|
|
_print_rrs_panel(rrs, components, config_slug)
|
|
else:
|
|
rrs, components = 0.0, {}
|
|
_console.print(f"\n[dim]RRS not computed: need ≥ 3 non-skipped scenarios, got {len(non_skipped)}[/dim]")
|
|
|
|
_console.print(f"\n[dim]Elapsed: {elapsed:.0f}s[/dim]\n")
|
|
|
|
if args.save:
|
|
save_dir = Path("runs/intraday_orb")
|
|
save_dir.mkdir(parents=True, exist_ok=True)
|
|
out_path = save_dir / f"{config_slug}_scenario_report.json"
|
|
payload = {
|
|
"config": str(config_path),
|
|
"scenarios_run": scenario_names,
|
|
"rrs": rrs,
|
|
"verdict": _rrs_verdict(rrs) if rrs > 0 else "N/A",
|
|
"components": components,
|
|
"results": scenario_results,
|
|
"elapsed_seconds": round(elapsed, 1),
|
|
}
|
|
out_path.write_text(json.dumps(payload, indent=2))
|
|
_console.print(f"[dim]Report saved → {out_path}[/dim]\n")
|
|
return 0
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(
|
|
prog="fithia2 intraday-scenario-test",
|
|
description="ORB strategy regime robustness test.",
|
|
)
|
|
parser.add_argument("--config", "-c", default=None, help="YAML config path or strategy slug")
|
|
parser.add_argument("--scenario", default=None, help="Run a single named scenario")
|
|
parser.add_argument("--group", default=None, help="Run a scenario group: regime, signal, quick, all")
|
|
parser.add_argument("--quick", action="store_true", help="Quick mode: run only bear_2022, bull_2023h2, oos_2026")
|
|
parser.add_argument("--start", default=_DEFAULT_START_DATE, help=f"Start of full data window (default: {_DEFAULT_START_DATE})")
|
|
parser.add_argument("--end", default=None, help="End of full data window (default: latest available)")
|
|
parser.add_argument("--save", action="store_true", help="Save JSON report to runs/intraday_orb")
|
|
parser.add_argument("--list", action="store_true", help="List all available scenarios and groups")
|
|
args = parser.parse_args()
|
|
raise SystemExit(asyncio.run(_async_main(args)))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|