fithia2/tests/unit/intraday/test_orb_research.py

from __future__ import annotations

import datetime as dt

import pytest

from libs.backtest.domain import (
    SplitResult,
    WalkForwardAggregate,
    WalkForwardGapStats,
    WalkForwardSummary,
)
from libs.intraday.domain import (
    BacktestParams,
    CacheParams,
    DayResult,
    IntradayConfig,
    ORBStrategyParams,
    OutputParams,
    UniverseParams,
)
from libs.intraday.orb_simulator import ORBSimulationState

import apps.intraday_bt.orb_research as orb_research
from apps.intraday_bt.orb_research import build_orb_research_context, compute_orb_overfit_score, compute_orbqs


def test_compute_orb_overfit_score_is_weighted_and_bounded() -> None:
    score, breakdown = compute_orb_overfit_score(
        {"retention_pct": 80.0},
        {"mean_sharpe": 1.2, "cv": 0.4},
        {"params": [{"plateau": 0.8}, {"plateau": 0.6}]},
        {"p_value": 0.04},
    )

    assert 0.0 <= score <= 100.0
    assert set(breakdown) == {
        "is_oos_retention",
        "wf_stability",
        "parameter_plateau",
        "candidate_permutation",
    }


def test_compute_orbqs_returns_breakdown_and_activity_penalty() -> None:
    train = SplitResult(
        run_id="train",
        trade_count=120,
        profit_factor=1.6,
        total_return_pct=18.0,
        annualized_return_pct=18.0,
        win_rate=0.52,
        max_drawdown_pct=8.0,
        sharpe_ratio=1.1,
        avg_gross_exposure_pct=20.0,
        avg_net_exposure_pct=20.0,
        days_in_market_pct=25.0,
    )
    valid = SplitResult(
        run_id="valid",
        trade_count=50,
        profit_factor=1.5,
        total_return_pct=12.0,
        annualized_return_pct=12.0,
        win_rate=0.5,
        max_drawdown_pct=7.0,
        sharpe_ratio=1.0,
        avg_gross_exposure_pct=20.0,
        avg_net_exposure_pct=20.0,
        days_in_market_pct=22.0,
    )
    test = SplitResult(
        run_id="test",
        trade_count=70,
        profit_factor=1.3,
        total_return_pct=8.0,
        annualized_return_pct=8.0,
        win_rate=0.48,
        max_drawdown_pct=6.0,
        sharpe_ratio=0.8,
        avg_gross_exposure_pct=20.0,
        avg_net_exposure_pct=20.0,
        days_in_market_pct=20.0,
    )
    wf_summary = WalkForwardSummary(
        train_days=252,
        test_days=63,
        step_days=63,
        fold_count=4,
        folds=[],
        train_aggregate=WalkForwardAggregate(mean_return_pct=12.0, median_return_pct=11.0),
        test_aggregate=WalkForwardAggregate(
            mean_return_pct=9.0,
            median_return_pct=8.0,
            worst_return_pct=2.0,
            positive_fold_rate_pct=75.0,
            mean_profit_factor=1.4,
            mean_max_drawdown_pct=7.0,
            mean_trade_count=30.0,
            mean_win_rate=0.5,
        ),
        gap_stats=WalkForwardGapStats(
            mean_train_test_return_gap_pct=25.0,
            worst_train_test_return_gap_pct=35.0,
            fold_return_cv=0.5,
        ),
        engine_reliability_ratio=1.0,
    )

    orbqs, breakdown = compute_orbqs(
        train,
        valid,
        test,
        wf_summary,
        {
            "bear_2022": {"sharpe_ratio": -0.4, "max_drawdown_pct": 12.0},
            "recovery_2023h1": {"sharpe_ratio": 0.9, "max_drawdown_pct": 9.0},
            "bull_2023h2": {"sharpe_ratio": 1.2, "max_drawdown_pct": 8.0},
            "oos_2026": {"sharpe_ratio": 0.8, "max_drawdown_pct": 6.0},
        },
        {
            "is_oos": {"retention_pct": 75.0},
            "walk_forward": {"mean_sharpe": 1.0, "cv": 0.5},
            "param_plateau": {"params": [{"plateau": 0.8}]},
            "permutation": {"p_value": 0.03},
        },
    )

    assert orbqs is not None
    assert 0.0 <= orbqs <= 100.0
    assert breakdown["activity_factor"] == 0.85
    assert "rqs_breakdown" in breakdown
    assert "wfqs_v2_breakdown" in breakdown
    assert "rrs_breakdown" in breakdown
    assert "overfit_breakdown" in breakdown


@pytest.mark.asyncio
async def test_build_orb_research_context_reuses_snapshot(tmp_path, monkeypatch) -> None:
    cache_dir = tmp_path / "intraday"
    config = IntradayConfig(
        strategy_mode="orb",
        orb_strategy=ORBStrategyParams(
            min_price=5.0,
            min_atr_14=0.5,
            min_avg_dollar_volume=1_000_000.0,
        ),
        universe=UniverseParams(source="midlarge"),
        backtest=BacktestParams(),
        cache=CacheParams(enabled=True, dir=str(cache_dir)),
        output=OutputParams(),
    )
    calls = {"fetch_daily": 0, "enrich": 0, "prescreen": 0}

    async def fake_resolve_universe(*args, **kwargs):
        return ["AAA", "BBB"]

    async def fake_get_trading_days(*args, **kwargs):
        return ["2024-01-02", "2024-01-03"]

    async def fake_fetch_daily(*args, **kwargs):
        calls["fetch_daily"] += 1
        return {
            "AAA": [{"date": "2024-01-02", "open": 10, "high": 11, "low": 9, "close": 10.5, "volume": 1000}],
            "BBB": [{"date": "2024-01-02", "open": 20, "high": 21, "low": 19, "close": 20.5, "volume": 2000}],
        }

    def fake_enrich(*args, **kwargs):
        calls["enrich"] += 1
        return {
            "AAA": {"2024-01-02": {"atr_14": 1.0}},
            "BBB": {"2024-01-02": {"atr_14": 1.2}},
        }

    def fake_prescreen(*args, **kwargs):
        calls["prescreen"] += 1
        return {
            "2024-01-02": ["AAA", "BBB"],
            "2024-01-03": ["AAA"],
        }

    monkeypatch.setattr(orb_research, "resolve_universe", fake_resolve_universe)
    monkeypatch.setattr(orb_research, "get_trading_days", fake_get_trading_days)
    monkeypatch.setattr(orb_research, "fetch_daily_bars_bulk", fake_fetch_daily)
    monkeypatch.setattr(orb_research, "enrich_daily_bars", fake_enrich)
    monkeypatch.setattr(orb_research, "orb_pre_screen_candidates", fake_prescreen)

    context = await build_orb_research_context(
        config,
        "2024-01-02",
        "2024-01-03",
        client=object(),
    )
    assert context.candidates["2024-01-02"] == ["AAA", "BBB"]
    assert calls == {"fetch_daily": 1, "enrich": 1, "prescreen": 1}

    snapshot_dir = cache_dir.with_name("orb_research")
    assert any(snapshot_dir.rglob("*.pkl.gz"))

    async def fail_fetch(*args, **kwargs):
        raise AssertionError("daily fetch should not run on snapshot hit")

    def fail_enrich(*args, **kwargs):
        raise AssertionError("enrichment should not run on snapshot hit")

    def fail_prescreen(*args, **kwargs):
        raise AssertionError("pre-screen should not run on snapshot hit")

    monkeypatch.setattr(orb_research, "fetch_daily_bars_bulk", fail_fetch)
    monkeypatch.setattr(orb_research, "enrich_daily_bars", fail_enrich)
    monkeypatch.setattr(orb_research, "orb_pre_screen_candidates", fail_prescreen)

    cached_context = await build_orb_research_context(
        config,
        "2024-01-02",
        "2024-01-03",
        client=object(),
    )
    assert cached_context.tickers == ["AAA", "BBB"]
    assert cached_context.trading_days == ["2024-01-02", "2024-01-03"]
    assert cached_context.candidates == context.candidates


@pytest.mark.asyncio
async def test_simulate_orb_period_reuses_period_metrics_cache(tmp_path, monkeypatch) -> None:
    cache_dir = tmp_path / "intraday"
    config = IntradayConfig(
        strategy_mode="orb",
        orb_strategy=ORBStrategyParams(),
        universe=UniverseParams(source="midlarge"),
        backtest=BacktestParams(),
        cache=CacheParams(enabled=True, dir=str(cache_dir)),
        output=OutputParams(),
    )
    eval_cache = orb_research.ORBPeriodMetricsCache(cache_dir.with_name("orb_eval"))
    context = orb_research.ORBResearchContext(
        config=config,
        tickers=["AAA"],
        trading_days=["2024-01-02"],
        daily_bars={"AAA": []},
        enrichment={},
        candidates={"2024-01-02": ["AAA"]},
        cache=None,
        daily_cache=None,
        eval_cache=eval_cache,
        tape_cache=None,
        oracle_url="http://localhost:18001",
        research_snapshot_key="snapshot_key",
    )
    calls = {"fetch": 0, "simulate": 0}

    async def fake_fetch_intraday(*args, **kwargs):
        calls["fetch"] += 1
        return {"AAA": {"2024-01-02": [{"timestamp": "2024-01-02T09:35:00-05:00"}]}}

    def fake_run_sim(*args, **kwargs):
        calls["simulate"] += 1
        return ([], None)

    monkeypatch.setattr(orb_research, "fetch_intraday_bulk", fake_fetch_intraday)
    monkeypatch.setattr(orb_research, "run_orb_simulation_with_state", fake_run_sim)

    metrics_first = await orb_research.simulate_orb_period(
        context,
        client=object(),
        orb_params=config.orb_strategy or ORBStrategyParams(),
        trading_days=["2024-01-02"],
        run_id="first",
    )
    assert calls == {"fetch": 1, "simulate": 1}
    assert metrics_first.run_id == "first"

    async def fail_fetch(*args, **kwargs):
        raise AssertionError("intraday fetch should not run on period cache hit")

    def fail_run(*args, **kwargs):
        raise AssertionError("simulation should not run on period cache hit")

    monkeypatch.setattr(orb_research, "fetch_intraday_bulk", fail_fetch)
    monkeypatch.setattr(orb_research, "run_orb_simulation_with_state", fail_run)

    metrics_second = await orb_research.simulate_orb_period(
        context,
        client=object(),
        orb_params=config.orb_strategy or ORBStrategyParams(),
        trading_days=["2024-01-02"],
        run_id="second",
    )
    assert metrics_second.run_id == "second"
    assert metrics_second.trading_days == metrics_first.trading_days


@pytest.mark.asyncio
async def test_simulate_orb_period_resumes_from_chunk_checkpoint(tmp_path, monkeypatch) -> None:
    cache_dir = tmp_path / "intraday"
    config = IntradayConfig(
        strategy_mode="orb",
        orb_strategy=ORBStrategyParams(initial_capital=10_000.0),
        universe=UniverseParams(source="midlarge"),
        backtest=BacktestParams(),
        cache=CacheParams(enabled=True, dir=str(cache_dir)),
        output=OutputParams(),
    )
    eval_cache = orb_research.ORBPeriodMetricsCache(cache_dir.with_name("orb_eval"))
    context = orb_research.ORBResearchContext(
        config=config,
        tickers=["AAA"],
        trading_days=["2024-01-02", "2024-01-03", "2024-01-04"],
        daily_bars={"AAA": []},
        enrichment={},
        candidates={
            "2024-01-02": ["AAA"],
            "2024-01-03": ["AAA"],
            "2024-01-04": ["AAA"],
        },
        cache=None,
        daily_cache=None,
        eval_cache=eval_cache,
        tape_cache=None,
        oracle_url="http://localhost:18001",
        research_snapshot_key="snapshot_key",
    )

    fetch_calls: list[str] = []
    run_states: list[float | None] = []
    first_run = {"attempt": True}

    async def flaky_fetch(chunk_candidates, *args, **kwargs):
        day = next(iter(chunk_candidates))
        fetch_calls.append(day)
        if first_run["attempt"] and day == "2024-01-03":
            raise RuntimeError("oracle timeout")
        return {
            day: {
                "AAA": [
                    {
                        "timestamp": f"{day}T09:35:00-05:00",
                        "open": 100.0,
                        "high": 101.0,
                        "low": 99.5,
                        "close": 100.5,
                        "volume": 1000.0,
                    }
                ]
            }
        }

    def fake_run_sim(
        all_intraday,
        trading_days,
        params,
        enrichment,
        ticker_sectors=None,
        state=None,
        progress_callback=None,
        **kwargs,
    ):
        run_states.append(state.equity if state is not None else None)
        day_results = [
            DayResult(date=day, daily_pnl=10.0, daily_return_pct=0.001)
            for day in trading_days
        ]
        next_equity = (state.equity if state is not None else params.initial_capital) + 10.0 * len(trading_days)
        return (
            day_results,
            ORBSimulationState(
                equity=next_equity,
                ticker_last_traded={"AAA": trading_days[-1]},
                settled_cash=None,
                pending_settlements=[],
            ),
        )

    monkeypatch.setattr(orb_research, "fetch_intraday_bulk", flaky_fetch)
    monkeypatch.setattr(orb_research, "run_orb_simulation_with_state", fake_run_sim)

    with pytest.raises(RuntimeError, match="oracle timeout"):
        await orb_research.simulate_orb_period(
            context,
            client=object(),
            orb_params=config.orb_strategy or ORBStrategyParams(),
            trading_days=context.trading_days,
            run_id="resume-test",
            max_pairs_per_chunk=1,
        )

    cache_key = eval_cache.build_key(
        research_snapshot_key="snapshot_key",
        orb_params=config.orb_strategy or ORBStrategyParams(),
        trading_days=context.trading_days,
        shuffle_candidates_seed=None,
    )
    checkpoint = eval_cache.load_checkpoint(cache_key)
    assert checkpoint is not None
    assert checkpoint["completed_chunks"] == 1

    first_run["attempt"] = False
    metrics = await orb_research.simulate_orb_period(
        context,
        client=object(),
        orb_params=config.orb_strategy or ORBStrategyParams(),
        trading_days=context.trading_days,
        run_id="resume-test",
        max_pairs_per_chunk=1,
    )

    assert fetch_calls == ["2024-01-02", "2024-01-03", "2024-01-03", "2024-01-04"]
    assert run_states == [None, 10010.0, 10020.0]
    assert metrics.trading_days == 3
    assert metrics.final_equity == 10030.0
    assert eval_cache.load(cache_key) is not None
    assert eval_cache.load_checkpoint(cache_key) is None


@pytest.mark.asyncio
async def test_simulate_orb_period_reuses_prepared_tape_for_new_params(tmp_path, monkeypatch) -> None:
    cache_dir = tmp_path / "intraday"
    config = IntradayConfig(
        strategy_mode="orb",
        orb_strategy=ORBStrategyParams(),
        universe=UniverseParams(source="midlarge"),
        backtest=BacktestParams(),
        cache=CacheParams(enabled=True, dir=str(cache_dir)),
        output=OutputParams(),
    )
    tape_cache = orb_research.ORBPreparedTapeStore(cache_dir.with_name("orb_tape"))
    context = orb_research.ORBResearchContext(
        config=config,
        tickers=["AAA"],
        trading_days=["2024-01-02"],
        daily_bars={"AAA": []},
        enrichment={},
        candidates={"2024-01-02": ["AAA"]},
        cache=None,
        daily_cache=None,
        eval_cache=None,
        tape_cache=tape_cache,
        oracle_url="http://localhost:18001",
        research_snapshot_key="snapshot_key",
    )
    calls = {"fetch": 0, "simulate": 0}

    async def fake_fetch_intraday(*args, **kwargs):
        calls["fetch"] += 1
        return {
            "2024-01-02": {
                "AAA": [
                    {
                        "timestamp": "2024-01-02T09:35:00-05:00",
                        "open": 100.0,
                        "high": 101.0,
                        "low": 99.5,
                        "close": 100.5,
                        "volume": 1000.0,
                    }
                ]
            }
        }

    def fake_run_sim(*args, **kwargs):
        calls["simulate"] += 1
        return ([DayResult(date="2024-01-02", daily_pnl=0.0, daily_return_pct=0.0)], ORBSimulationState(equity=10_000.0))

    monkeypatch.setattr(orb_research, "fetch_intraday_bulk", fake_fetch_intraday)
    monkeypatch.setattr(orb_research, "run_orb_simulation_with_state", fake_run_sim)

    await orb_research.simulate_orb_period(
        context,
        client=object(),
        orb_params=ORBStrategyParams(atr_stop_multiplier=1.0),
        trading_days=["2024-01-02"],
        run_id="tape-first",
    )
    assert calls == {"fetch": 1, "simulate": 1}
    assert any((cache_dir.with_name("orb_tape")).rglob("*.pkl.gz"))

    async def fail_fetch(*args, **kwargs):
        raise AssertionError("raw intraday fetch should not run on tape hit")

    monkeypatch.setattr(orb_research, "fetch_intraday_bulk", fail_fetch)

    await orb_research.simulate_orb_period(
        context,
        client=object(),
        orb_params=ORBStrategyParams(atr_stop_multiplier=1.25),
        trading_days=["2024-01-02"],
        run_id="tape-second",
    )
    assert calls == {"fetch": 1, "simulate": 2}