from __future__ import annotations import datetime as dt import pytest from libs.backtest.domain import ( SplitResult, WalkForwardAggregate, WalkForwardGapStats, WalkForwardSummary, ) from libs.intraday.domain import ( BacktestParams, CacheParams, DayResult, IntradayConfig, ORBStrategyParams, OutputParams, UniverseParams, ) from libs.intraday.orb_simulator import ORBSimulationState import apps.intraday_bt.orb_research as orb_research from apps.intraday_bt.orb_research import build_orb_research_context, compute_orb_overfit_score, compute_orbqs def test_compute_orb_overfit_score_is_weighted_and_bounded() -> None: score, breakdown = compute_orb_overfit_score( {"retention_pct": 80.0}, {"mean_sharpe": 1.2, "cv": 0.4}, {"params": [{"plateau": 0.8}, {"plateau": 0.6}]}, {"p_value": 0.04}, ) assert 0.0 <= score <= 100.0 assert set(breakdown) == { "is_oos_retention", "wf_stability", "parameter_plateau", "candidate_permutation", } def test_compute_orbqs_returns_breakdown_and_activity_penalty() -> None: train = SplitResult( run_id="train", trade_count=120, profit_factor=1.6, total_return_pct=18.0, annualized_return_pct=18.0, win_rate=0.52, max_drawdown_pct=8.0, sharpe_ratio=1.1, avg_gross_exposure_pct=20.0, avg_net_exposure_pct=20.0, days_in_market_pct=25.0, ) valid = SplitResult( run_id="valid", trade_count=50, profit_factor=1.5, total_return_pct=12.0, annualized_return_pct=12.0, win_rate=0.5, max_drawdown_pct=7.0, sharpe_ratio=1.0, avg_gross_exposure_pct=20.0, avg_net_exposure_pct=20.0, days_in_market_pct=22.0, ) test = SplitResult( run_id="test", trade_count=70, profit_factor=1.3, total_return_pct=8.0, annualized_return_pct=8.0, win_rate=0.48, max_drawdown_pct=6.0, sharpe_ratio=0.8, avg_gross_exposure_pct=20.0, avg_net_exposure_pct=20.0, days_in_market_pct=20.0, ) wf_summary = WalkForwardSummary( train_days=252, test_days=63, step_days=63, fold_count=4, folds=[], train_aggregate=WalkForwardAggregate(mean_return_pct=12.0, median_return_pct=11.0), test_aggregate=WalkForwardAggregate( mean_return_pct=9.0, median_return_pct=8.0, worst_return_pct=2.0, positive_fold_rate_pct=75.0, mean_profit_factor=1.4, mean_max_drawdown_pct=7.0, mean_trade_count=30.0, mean_win_rate=0.5, ), gap_stats=WalkForwardGapStats( mean_train_test_return_gap_pct=25.0, worst_train_test_return_gap_pct=35.0, fold_return_cv=0.5, ), engine_reliability_ratio=1.0, ) orbqs, breakdown = compute_orbqs( train, valid, test, wf_summary, { "bear_2022": {"sharpe_ratio": -0.4, "max_drawdown_pct": 12.0}, "recovery_2023h1": {"sharpe_ratio": 0.9, "max_drawdown_pct": 9.0}, "bull_2023h2": {"sharpe_ratio": 1.2, "max_drawdown_pct": 8.0}, "oos_2026": {"sharpe_ratio": 0.8, "max_drawdown_pct": 6.0}, }, { "is_oos": {"retention_pct": 75.0}, "walk_forward": {"mean_sharpe": 1.0, "cv": 0.5}, "param_plateau": {"params": [{"plateau": 0.8}]}, "permutation": {"p_value": 0.03}, }, ) assert orbqs is not None assert 0.0 <= orbqs <= 100.0 assert breakdown["activity_factor"] == 0.85 assert "rqs_breakdown" in breakdown assert "wfqs_v2_breakdown" in breakdown assert "rrs_breakdown" in breakdown assert "overfit_breakdown" in breakdown @pytest.mark.asyncio async def test_build_orb_research_context_reuses_snapshot(tmp_path, monkeypatch) -> None: cache_dir = tmp_path / "intraday" config = IntradayConfig( strategy_mode="orb", orb_strategy=ORBStrategyParams( min_price=5.0, min_atr_14=0.5, min_avg_dollar_volume=1_000_000.0, ), universe=UniverseParams(source="midlarge"), backtest=BacktestParams(), cache=CacheParams(enabled=True, dir=str(cache_dir)), output=OutputParams(), ) calls = {"fetch_daily": 0, "enrich": 0, "prescreen": 0} async def fake_resolve_universe(*args, **kwargs): return ["AAA", "BBB"] async def fake_get_trading_days(*args, **kwargs): return ["2024-01-02", "2024-01-03"] async def fake_fetch_daily(*args, **kwargs): calls["fetch_daily"] += 1 return { "AAA": [{"date": "2024-01-02", "open": 10, "high": 11, "low": 9, "close": 10.5, "volume": 1000}], "BBB": [{"date": "2024-01-02", "open": 20, "high": 21, "low": 19, "close": 20.5, "volume": 2000}], } def fake_enrich(*args, **kwargs): calls["enrich"] += 1 return { "AAA": {"2024-01-02": {"atr_14": 1.0}}, "BBB": {"2024-01-02": {"atr_14": 1.2}}, } def fake_prescreen(*args, **kwargs): calls["prescreen"] += 1 return { "2024-01-02": ["AAA", "BBB"], "2024-01-03": ["AAA"], } monkeypatch.setattr(orb_research, "resolve_universe", fake_resolve_universe) monkeypatch.setattr(orb_research, "get_trading_days", fake_get_trading_days) monkeypatch.setattr(orb_research, "fetch_daily_bars_bulk", fake_fetch_daily) monkeypatch.setattr(orb_research, "enrich_daily_bars", fake_enrich) monkeypatch.setattr(orb_research, "orb_pre_screen_candidates", fake_prescreen) context = await build_orb_research_context( config, "2024-01-02", "2024-01-03", client=object(), ) assert context.candidates["2024-01-02"] == ["AAA", "BBB"] assert calls == {"fetch_daily": 1, "enrich": 1, "prescreen": 1} snapshot_dir = cache_dir.with_name("orb_research") assert any(snapshot_dir.rglob("*.pkl.gz")) async def fail_fetch(*args, **kwargs): raise AssertionError("daily fetch should not run on snapshot hit") def fail_enrich(*args, **kwargs): raise AssertionError("enrichment should not run on snapshot hit") def fail_prescreen(*args, **kwargs): raise AssertionError("pre-screen should not run on snapshot hit") monkeypatch.setattr(orb_research, "fetch_daily_bars_bulk", fail_fetch) monkeypatch.setattr(orb_research, "enrich_daily_bars", fail_enrich) monkeypatch.setattr(orb_research, "orb_pre_screen_candidates", fail_prescreen) cached_context = await build_orb_research_context( config, "2024-01-02", "2024-01-03", client=object(), ) assert cached_context.tickers == ["AAA", "BBB"] assert cached_context.trading_days == ["2024-01-02", "2024-01-03"] assert cached_context.candidates == context.candidates @pytest.mark.asyncio async def test_simulate_orb_period_reuses_period_metrics_cache(tmp_path, monkeypatch) -> None: cache_dir = tmp_path / "intraday" config = IntradayConfig( strategy_mode="orb", orb_strategy=ORBStrategyParams(), universe=UniverseParams(source="midlarge"), backtest=BacktestParams(), cache=CacheParams(enabled=True, dir=str(cache_dir)), output=OutputParams(), ) eval_cache = orb_research.ORBPeriodMetricsCache(cache_dir.with_name("orb_eval")) context = orb_research.ORBResearchContext( config=config, tickers=["AAA"], trading_days=["2024-01-02"], daily_bars={"AAA": []}, enrichment={}, candidates={"2024-01-02": ["AAA"]}, cache=None, daily_cache=None, eval_cache=eval_cache, tape_cache=None, oracle_url="http://localhost:18001", research_snapshot_key="snapshot_key", ) calls = {"fetch": 0, "simulate": 0} async def fake_fetch_intraday(*args, **kwargs): calls["fetch"] += 1 return {"AAA": {"2024-01-02": [{"timestamp": "2024-01-02T09:35:00-05:00"}]}} def fake_run_sim(*args, **kwargs): calls["simulate"] += 1 return ([], None) monkeypatch.setattr(orb_research, "fetch_intraday_bulk", fake_fetch_intraday) monkeypatch.setattr(orb_research, "run_orb_simulation_with_state", fake_run_sim) metrics_first = await orb_research.simulate_orb_period( context, client=object(), orb_params=config.orb_strategy or ORBStrategyParams(), trading_days=["2024-01-02"], run_id="first", ) assert calls == {"fetch": 1, "simulate": 1} assert metrics_first.run_id == "first" async def fail_fetch(*args, **kwargs): raise AssertionError("intraday fetch should not run on period cache hit") def fail_run(*args, **kwargs): raise AssertionError("simulation should not run on period cache hit") monkeypatch.setattr(orb_research, "fetch_intraday_bulk", fail_fetch) monkeypatch.setattr(orb_research, "run_orb_simulation_with_state", fail_run) metrics_second = await orb_research.simulate_orb_period( context, client=object(), orb_params=config.orb_strategy or ORBStrategyParams(), trading_days=["2024-01-02"], run_id="second", ) assert metrics_second.run_id == "second" assert metrics_second.trading_days == metrics_first.trading_days @pytest.mark.asyncio async def test_simulate_orb_period_resumes_from_chunk_checkpoint(tmp_path, monkeypatch) -> None: cache_dir = tmp_path / "intraday" config = IntradayConfig( strategy_mode="orb", orb_strategy=ORBStrategyParams(initial_capital=10_000.0), universe=UniverseParams(source="midlarge"), backtest=BacktestParams(), cache=CacheParams(enabled=True, dir=str(cache_dir)), output=OutputParams(), ) eval_cache = orb_research.ORBPeriodMetricsCache(cache_dir.with_name("orb_eval")) context = orb_research.ORBResearchContext( config=config, tickers=["AAA"], trading_days=["2024-01-02", "2024-01-03", "2024-01-04"], daily_bars={"AAA": []}, enrichment={}, candidates={ "2024-01-02": ["AAA"], "2024-01-03": ["AAA"], "2024-01-04": ["AAA"], }, cache=None, daily_cache=None, eval_cache=eval_cache, tape_cache=None, oracle_url="http://localhost:18001", research_snapshot_key="snapshot_key", ) fetch_calls: list[str] = [] run_states: list[float | None] = [] first_run = {"attempt": True} async def flaky_fetch(chunk_candidates, *args, **kwargs): day = next(iter(chunk_candidates)) fetch_calls.append(day) if first_run["attempt"] and day == "2024-01-03": raise RuntimeError("oracle timeout") return { day: { "AAA": [ { "timestamp": f"{day}T09:35:00-05:00", "open": 100.0, "high": 101.0, "low": 99.5, "close": 100.5, "volume": 1000.0, } ] } } def fake_run_sim( all_intraday, trading_days, params, enrichment, ticker_sectors=None, state=None, progress_callback=None, **kwargs, ): run_states.append(state.equity if state is not None else None) day_results = [ DayResult(date=day, daily_pnl=10.0, daily_return_pct=0.001) for day in trading_days ] next_equity = (state.equity if state is not None else params.initial_capital) + 10.0 * len(trading_days) return ( day_results, ORBSimulationState( equity=next_equity, ticker_last_traded={"AAA": trading_days[-1]}, settled_cash=None, pending_settlements=[], ), ) monkeypatch.setattr(orb_research, "fetch_intraday_bulk", flaky_fetch) monkeypatch.setattr(orb_research, "run_orb_simulation_with_state", fake_run_sim) with pytest.raises(RuntimeError, match="oracle timeout"): await orb_research.simulate_orb_period( context, client=object(), orb_params=config.orb_strategy or ORBStrategyParams(), trading_days=context.trading_days, run_id="resume-test", max_pairs_per_chunk=1, ) cache_key = eval_cache.build_key( research_snapshot_key="snapshot_key", orb_params=config.orb_strategy or ORBStrategyParams(), trading_days=context.trading_days, shuffle_candidates_seed=None, ) checkpoint = eval_cache.load_checkpoint(cache_key) assert checkpoint is not None assert checkpoint["completed_chunks"] == 1 first_run["attempt"] = False metrics = await orb_research.simulate_orb_period( context, client=object(), orb_params=config.orb_strategy or ORBStrategyParams(), trading_days=context.trading_days, run_id="resume-test", max_pairs_per_chunk=1, ) assert fetch_calls == ["2024-01-02", "2024-01-03", "2024-01-03", "2024-01-04"] assert run_states == [None, 10010.0, 10020.0] assert metrics.trading_days == 3 assert metrics.final_equity == 10030.0 assert eval_cache.load(cache_key) is not None assert eval_cache.load_checkpoint(cache_key) is None @pytest.mark.asyncio async def test_simulate_orb_period_reuses_prepared_tape_for_new_params(tmp_path, monkeypatch) -> None: cache_dir = tmp_path / "intraday" config = IntradayConfig( strategy_mode="orb", orb_strategy=ORBStrategyParams(), universe=UniverseParams(source="midlarge"), backtest=BacktestParams(), cache=CacheParams(enabled=True, dir=str(cache_dir)), output=OutputParams(), ) tape_cache = orb_research.ORBPreparedTapeStore(cache_dir.with_name("orb_tape")) context = orb_research.ORBResearchContext( config=config, tickers=["AAA"], trading_days=["2024-01-02"], daily_bars={"AAA": []}, enrichment={}, candidates={"2024-01-02": ["AAA"]}, cache=None, daily_cache=None, eval_cache=None, tape_cache=tape_cache, oracle_url="http://localhost:18001", research_snapshot_key="snapshot_key", ) calls = {"fetch": 0, "simulate": 0} async def fake_fetch_intraday(*args, **kwargs): calls["fetch"] += 1 return { "2024-01-02": { "AAA": [ { "timestamp": "2024-01-02T09:35:00-05:00", "open": 100.0, "high": 101.0, "low": 99.5, "close": 100.5, "volume": 1000.0, } ] } } def fake_run_sim(*args, **kwargs): calls["simulate"] += 1 return ([DayResult(date="2024-01-02", daily_pnl=0.0, daily_return_pct=0.0)], ORBSimulationState(equity=10_000.0)) monkeypatch.setattr(orb_research, "fetch_intraday_bulk", fake_fetch_intraday) monkeypatch.setattr(orb_research, "run_orb_simulation_with_state", fake_run_sim) await orb_research.simulate_orb_period( context, client=object(), orb_params=ORBStrategyParams(atr_stop_multiplier=1.0), trading_days=["2024-01-02"], run_id="tape-first", ) assert calls == {"fetch": 1, "simulate": 1} assert any((cache_dir.with_name("orb_tape")).rglob("*.pkl.gz")) async def fail_fetch(*args, **kwargs): raise AssertionError("raw intraday fetch should not run on tape hit") monkeypatch.setattr(orb_research, "fetch_intraday_bulk", fail_fetch) await orb_research.simulate_orb_period( context, client=object(), orb_params=ORBStrategyParams(atr_stop_multiplier=1.25), trading_days=["2024-01-02"], run_id="tape-second", ) assert calls == {"fetch": 1, "simulate": 2}