fithia2/apps/tools/idle_capital_ideas_probe.py

"""Research probe for non-parking idle-capital ideas.

This tool does not change the core backtest engine.

Supported probes:
  - overnight: invest idle cash near the close and unwind at next open
  - pairs: simple market-neutral pair-trading sanity checks
  - weekday-collateral: intraday weekday sleeve funded by buying power or reserved cash
"""
from __future__ import annotations

import argparse
import datetime as dt
import json
import math
import statistics
from dataclasses import dataclass
from itertools import product
from typing import Any

from apps.backtester.run import BacktestRunner, _build_merged_snapshot_store, load_manifest, resolve_config
from apps.tools.put_spread_overlay_probe import (
    _build_runner,
    _compute_probe_metrics,
    _parse_date,
)
from libs.backtest.domain import DailyPortfolioState
from libs.backtest.metrics import (
    compute_max_drawdown_pct,
    compute_sharpe_ratio,
    compute_total_return_pct,
)
from libs.common.logging import configure_logging


@dataclass(frozen=True)
class PairParams:
    lookback: int = 60
    enter_z: float = 2.0
    exit_z: float = 0.5
    max_hold: int = 20


def _simulate_overnight_overlay(
    curve: list[DailyPortfolioState],
    store: Any,
    *,
    symbol_prefix: str,
    min_cash_ratio: float,
    require_no_open_positions: bool,
) -> tuple[list[DailyPortfolioState], int, float]:
    dates = [state.date for state in curve]
    realized_pnl = 0.0
    adjusted_curve: list[DailyPortfolioState] = []
    traded_nights = 0

    for idx, state in enumerate(curve):
        adjusted_curve.append(
            state.model_copy(update={"equity": state.equity + (realized_pnl if idx > 0 else 0.0)})
        )
        if idx >= len(dates) - 1 or state.equity <= 0:
            continue
        if state.cash_available / state.equity < min_cash_ratio:
            continue
        if require_no_open_positions and state.open_positions:
            continue

        next_date = dates[idx + 1]
        curr_macro = store.get_macro_for_date(state.date)
        next_macro = store.get_macro_for_date(next_date)
        close_px = curr_macro.get(f"{symbol_prefix}_close")
        open_px = next_macro.get(f"{symbol_prefix}_open")
        if not close_px or not open_px or close_px <= 0 or open_px <= 0:
            continue

        overnight_ret = float(open_px) / float(close_px) - 1.0
        realized_pnl += state.cash_available * overnight_ret
        traded_nights += 1

    return adjusted_curve, traded_nights, round(realized_pnl, 2)


def _run_overnight_probe(args: argparse.Namespace) -> list[dict[str, Any]]:
    start_date = _parse_date(args.start)
    end_date = _parse_date(args.end, is_end=True)
    rows: list[dict[str, Any]] = []
    for config_path in args.config:
        base_runner, store = _build_runner(
            config_path,
            start_date,
            end_date,
            args.capital,
            parking_preset=None,
        )
        base_curve = base_runner._equity_curve
        base_metrics = _compute_probe_metrics(base_curve)
        ref_curve = None
        ref_metrics = None
        if args.parking_preset:
            ref_runner, _ = _build_runner(
                config_path,
                start_date,
                end_date,
                args.capital,
                parking_preset=args.parking_preset,
            )
            ref_curve = ref_runner._equity_curve
            ref_metrics = _compute_probe_metrics(ref_curve)

        adjusted_curve, traded_nights, overlay_pnl = _simulate_overnight_overlay(
            base_curve,
            store,
            symbol_prefix=args.symbol.lower(),
            min_cash_ratio=args.min_cash_pct,
            require_no_open_positions=args.require_no_open_positions,
        )
        overlay_metrics = _compute_probe_metrics(adjusted_curve)
        row: dict[str, Any] = {
            "config": config_path,
            "symbol": args.symbol.upper(),
            "base_no_parking": base_metrics.__dict__,
            "overlay": overlay_metrics.__dict__,
            "overlay_delta_return_pct": round(
                overlay_metrics.total_return_pct - base_metrics.total_return_pct,
                2,
            ),
            "overlay_delta_dd_pct": round(
                overlay_metrics.max_drawdown_pct - base_metrics.max_drawdown_pct,
                2,
            ),
            "overlay_traded_nights": traded_nights,
            "overlay_realized_pnl": overlay_pnl,
        }
        if ref_metrics is not None:
            row["parking_reference"] = ref_metrics.__dict__
            row["overlay_vs_parking_return_pct"] = round(
                overlay_metrics.total_return_pct - ref_metrics.total_return_pct,
                2,
            )
            row["overlay_vs_parking_dd_pct"] = round(
                overlay_metrics.max_drawdown_pct - ref_metrics.max_drawdown_pct,
                2,
            )
        rows.append(row)
    return rows


def _parse_pair(value: str) -> tuple[str, str]:
    left, right = value.split("/", 1)
    return left.strip().upper(), right.strip().upper()


def _pair_curve(
    *,
    store: Any,
    left_symbol: str,
    right_symbol: str,
    start_date: dt.date,
    end_date: dt.date,
    params: PairParams,
    capital: float,
) -> dict[str, Any] | None:
    dates: list[dt.date] = []
    left_closes: list[float] = []
    right_closes: list[float] = []
    for date in store.all_trading_days():
        if date < start_date or date > end_date:
            continue
        left_bar = store.get_bar(left_symbol, date)
        right_bar = store.get_bar(right_symbol, date)
        if not left_bar or not right_bar:
            continue
        dates.append(date)
        left_closes.append(float(left_bar["close"]))
        right_closes.append(float(right_bar["close"]))

    if len(dates) < params.lookback + 5:
        return None

    equity = capital
    curve: list[DailyPortfolioState] = []
    open_position: dict[str, float | int] | None = None
    prev_left = None
    prev_right = None
    trades = 0

    for idx, date in enumerate(dates):
        left_close = left_closes[idx]
        right_close = right_closes[idx]
        if open_position is not None and prev_left and prev_right:
            left_ret = left_close / prev_left - 1.0
            right_ret = right_close / prev_right - 1.0
            pnl_fraction = (
                float(open_position["sign_left"]) * 0.5 * left_ret
                + float(open_position["sign_right"]) * 0.5 * right_ret
            )
            equity *= 1.0 + pnl_fraction
            open_position["days"] = int(open_position["days"]) + 1

        curve.append(
            DailyPortfolioState(
                date=date,
                equity=equity,
                sizing_equity=equity,
                cash_available=equity,
                gross_exposure=100.0 if open_position else 0.0,
                net_exposure=0.0,
                reserved_risk_budget=0.0,
                unrealized_pnl=0.0,
                realized_pnl=0.0,
                open_positions=["pair"] if open_position else [],
                daily_new_risk_used=0.0,
                peak_equity=max(equity, curve[-1].peak_equity if curve else equity),
                current_drawdown_pct=0.0,
            )
        )

        if idx >= params.lookback:
            window = [
                math.log(left_closes[j] / right_closes[j])
                for j in range(idx - params.lookback, idx)
            ]
            mean_spread = statistics.mean(window)
            spread_std = statistics.pstdev(window)
            if spread_std > 1e-9:
                zscore = (math.log(left_close / right_close) - mean_spread) / spread_std
                if open_position is None:
                    if zscore > params.enter_z:
                        open_position = {"sign_left": -1.0, "sign_right": 1.0, "days": 0}
                        trades += 1
                    elif zscore < -params.enter_z:
                        open_position = {"sign_left": 1.0, "sign_right": -1.0, "days": 0}
                        trades += 1
                elif abs(zscore) < params.exit_z or int(open_position["days"]) >= params.max_hold:
                    open_position = None

        prev_left = left_close
        prev_right = right_close

    return {
        "pair": f"{left_symbol}/{right_symbol}",
        "params": params.__dict__,
        "total_return_pct": round(compute_total_return_pct(curve) or 0.0, 2),
        "max_drawdown_pct": round(compute_max_drawdown_pct(curve) or 0.0, 2),
        "sharpe_ratio": round(compute_sharpe_ratio(curve) or 0.0, 3),
        "trades": trades,
    }


def _run_pairs_probe(args: argparse.Namespace) -> list[dict[str, Any]]:
    start_date = _parse_date(args.start)
    end_date = _parse_date(args.end, is_end=True)
    base_runner, store = _build_runner(
        args.config,
        start_date,
        end_date,
        args.capital,
        parking_preset=None,
    )
    del base_runner

    pairs = [_parse_pair(value) for value in args.pair]
    rows: list[dict[str, Any]] = []
    if args.grid:
        grid = list(product([40, 60], [1.5, 2.0], [0.25, 0.5], [10, 20]))
        for left_symbol, right_symbol in pairs:
            best: dict[str, Any] | None = None
            for lookback, enter_z, exit_z, max_hold in grid:
                row = _pair_curve(
                    store=store,
                    left_symbol=left_symbol,
                    right_symbol=right_symbol,
                    start_date=start_date,
                    end_date=end_date,
                    params=PairParams(
                        lookback=lookback,
                        enter_z=enter_z,
                        exit_z=exit_z,
                        max_hold=max_hold,
                    ),
                    capital=args.capital,
                )
                if row is None:
                    continue
                if best is None or (
                    row["total_return_pct"],
                    -row["max_drawdown_pct"],
                    row["sharpe_ratio"],
                ) > (
                    best["total_return_pct"],
                    -best["max_drawdown_pct"],
                    best["sharpe_ratio"],
                ):
                    best = row
            if best is not None:
                rows.append(best)
        return rows

    params = PairParams(
        lookback=args.lookback,
        enter_z=args.enter_z,
        exit_z=args.exit_z,
        max_hold=args.max_hold,
    )
    for left_symbol, right_symbol in pairs:
        row = _pair_curve(
            store=store,
            left_symbol=left_symbol,
            right_symbol=right_symbol,
            start_date=start_date,
            end_date=end_date,
            params=params,
            capital=args.capital,
        )
        if row is not None:
            rows.append(row)
    return rows


def _run_micro_event_probe(args: argparse.Namespace) -> list[dict[str, Any]]:
    start_date = _parse_date(args.start)
    end_date = _parse_date(args.end, is_end=True)
    manifest = load_manifest(args.config)
    base_config = resolve_config(manifest)
    store = _build_merged_snapshot_store(
        manifest,
        base_config,
        snapshot_dir_override=None,
    ).slice_by_date_range(start_date, end_date)

    def _run_config(config: Any) -> tuple[ProbeMetrics, Any]:
        runner = BacktestRunner(
            manifest=manifest,
            config=config,
            store=store,
            initial_equity=args.capital,
            split_name="idle_micro_event_probe",
        )
        runner.run(output_root=None)
        return _compute_probe_metrics(runner._equity_curve), runner

    base_metrics, base_runner = _run_config(base_config)
    row: dict[str, Any] = {
        "config": args.config,
        "mode": args.mode,
        "engine_ids": list(args.engine_id),
        "base": base_metrics.__dict__,
    }

    variant_config = resolve_config(manifest)
    selected_ids = set(args.engine_id)
    if args.mode == "clone":
        lookup = {engine.engine_id: engine for engine in variant_config.strategy_engines}
        new_engines = list(variant_config.strategy_engines)
        for engine_id in args.engine_id:
            source = lookup[engine_id]
            new_engines.append(
                source.model_copy(
                    update={
                        "engine_id": f"{engine_id}_micro",
                        "selection_priority": args.selection_priority,
                        "capital_bucket_id": args.bucket_id,
                        "capital_bucket_allocation_pct": args.bucket_allocation_pct,
                        "engine_risk_budget_pct": min(float(source.engine_risk_budget_pct), 0.20),
                        "per_trade_risk_pct_override": (
                            args.per_trade_risk_pct
                            if args.per_trade_risk_pct is not None
                            else source.per_trade_risk_pct_override
                        ),
                        "residual_reserve_selected": False,
                        "recycle_on_cash_block": False,
                    }
                )
            )
        variant_config.strategy_engines = new_engines
        tracked_engine_ids = {f"{engine_id}_micro" for engine_id in args.engine_id}
    else:
        new_engines = []
        for engine in variant_config.strategy_engines:
            if engine.engine_id in selected_ids:
                new_engines.append(
                    engine.model_copy(
                        update={
                            "capital_bucket_id": args.bucket_id,
                            "capital_bucket_allocation_pct": args.bucket_allocation_pct,
                            "per_trade_risk_pct_override": (
                                args.per_trade_risk_pct
                                if args.per_trade_risk_pct is not None
                                else engine.per_trade_risk_pct_override
                            ),
                        }
                    )
                )
            else:
                new_engines.append(engine)
        variant_config.strategy_engines = new_engines
        tracked_engine_ids = selected_ids

    variant_metrics, variant_runner = _run_config(variant_config)
    row["variant"] = variant_metrics.__dict__
    row["variant_delta_return_pct"] = round(
        variant_metrics.total_return_pct - base_metrics.total_return_pct,
        2,
    )
    row["variant_delta_dd_pct"] = round(
        variant_metrics.max_drawdown_pct - base_metrics.max_drawdown_pct,
        2,
    )
    row["variant_trade_count"] = len(variant_runner._closed_trades)
    row["variant_tracked_trade_count"] = sum(
        1 for trade in variant_runner._closed_trades if trade.engine_id in tracked_engine_ids
    )
    row["variant_tracked_pnl"] = round(
        sum(float(trade.net_pnl) for trade in variant_runner._closed_trades if trade.engine_id in tracked_engine_ids),
        2,
    )

    if args.parking_preset:
        ref_runner, _ = _build_runner(
            args.config,
            start_date,
            end_date,
            args.capital,
            parking_preset=args.parking_preset,
        )
        ref_metrics = _compute_probe_metrics(ref_runner._equity_curve)
        row["parking_reference"] = ref_metrics.__dict__
        row["variant_vs_parking_return_pct"] = round(
            variant_metrics.total_return_pct - ref_metrics.total_return_pct,
            2,
        )
        row["variant_vs_parking_dd_pct"] = round(
            variant_metrics.max_drawdown_pct - ref_metrics.max_drawdown_pct,
            2,
        )

    return [row]


def _weekday_filter_allows(store: Any, date: dt.date, filter_name: str) -> bool:
    macro = store.get_macro_for_date(date)
    if filter_name == "none":
        return True
    if filter_name == "trend50":
        close = macro.get("spy_close")
        sma = macro.get("spy_sma_50")
        return close is not None and sma is not None and float(close) >= float(sma)
    if filter_name == "mom20":
        mom = macro.get("spy_mom_20")
        return mom is not None and float(mom) > 0.0
    if filter_name == "vol15":
        vol = macro.get("spy_vol_20")
        return vol is not None and float(vol) < 0.15
    if filter_name == "trend50_vol15":
        close = macro.get("spy_close")
        sma = macro.get("spy_sma_50")
        vol = macro.get("spy_vol_20")
        return (
            close is not None
            and sma is not None
            and vol is not None
            and float(close) >= float(sma)
            and float(vol) < 0.15
        )
    raise ValueError(f"Unsupported weekday filter: {filter_name}")


def _simulate_weekday_collateral_overlay(
    curve: list[DailyPortfolioState],
    store: Any,
    *,
    symbol: str,
    weekday: int,
    filter_name: str,
    capital_source: str,
    reserve_pct: float,
) -> tuple[list[DailyPortfolioState], int, float, dict[int, float]]:
    realized_pnl = 0.0
    adjusted_curve: list[DailyPortfolioState] = []
    traded_days = 0
    pnl_by_year: dict[int, float] = {}

    for idx, state in enumerate(curve):
        adjusted_curve.append(
            state.model_copy(update={"equity": state.equity + (realized_pnl if idx > 0 else 0.0)})
        )
        if state.date.weekday() != weekday:
            continue
        if not _weekday_filter_allows(store, state.date, filter_name):
            continue
        bar = store.get_bar(symbol, state.date)
        if not bar:
            continue
        open_px = bar.get("open")
        close_px = bar.get("close")
        if not open_px or not close_px or open_px <= 0 or close_px <= 0:
            continue
        if capital_source == "cash_available":
            trade_capital = float(state.cash_available)
        else:
            trade_capital = float(state.equity) * reserve_pct
        if trade_capital <= 0:
            continue
        intraday_ret = float(close_px) / float(open_px) - 1.0
        day_pnl = trade_capital * intraday_ret
        realized_pnl += day_pnl
        traded_days += 1
        pnl_by_year[state.date.year] = pnl_by_year.get(state.date.year, 0.0) + day_pnl

    return adjusted_curve, traded_days, round(realized_pnl, 2), pnl_by_year


def _run_weekday_collateral_probe(args: argparse.Namespace) -> list[dict[str, Any]]:
    start_date = _parse_date(args.start)
    end_date = _parse_date(args.end, is_end=True)
    rows: list[dict[str, Any]] = []
    for config_path in args.config:
        if args.capital_source == "reserve_pct":
            manifest = load_manifest(config_path)
            config = resolve_config(manifest)
            if args.parking_preset:
                config.risk.cash_parking_enabled = True
                config.risk.cash_parking_preset = args.parking_preset
                config.risk.apply_parking_preset()
            config.risk.cash_parking_reserve_pct = args.reserve_pct
            store = _build_merged_snapshot_store(
                manifest,
                config,
                snapshot_dir_override=None,
            ).slice_by_date_range(start_date, end_date)
            base_runner = BacktestRunner(
                manifest=manifest,
                config=config,
                store=store,
                initial_equity=args.capital,
                split_name="weekday_collateral_probe",
            )
            base_runner.run(output_root=None)
        else:
            base_runner, store = _build_runner(
                config_path,
                start_date,
                end_date,
                args.capital,
                parking_preset=args.parking_preset,
            )
        base_curve = base_runner._equity_curve
        base_metrics = _compute_probe_metrics(base_curve)
        (
            adjusted_curve,
            traded_days,
            overlay_pnl,
            pnl_by_year,
        ) = _simulate_weekday_collateral_overlay(
            base_curve,
            store,
            symbol=args.symbol.upper(),
            weekday=args.weekday,
            filter_name=args.filter,
            capital_source=args.capital_source,
            reserve_pct=args.reserve_pct,
        )
        overlay_metrics = _compute_probe_metrics(adjusted_curve)
        row: dict[str, Any] = {
            "config": config_path,
            "parking_preset": args.parking_preset,
            "symbol": args.symbol.upper(),
            "weekday": args.weekday,
            "filter": args.filter,
            "capital_source": args.capital_source,
            "reserve_pct": args.reserve_pct,
            "base": base_metrics.__dict__,
            "overlay": overlay_metrics.__dict__,
            "overlay_delta_return_pct": round(
                overlay_metrics.total_return_pct - base_metrics.total_return_pct,
                2,
            ),
            "overlay_delta_dd_pct": round(
                overlay_metrics.max_drawdown_pct - base_metrics.max_drawdown_pct,
                2,
            ),
            "overlay_traded_days": traded_days,
            "overlay_realized_pnl": overlay_pnl,
            "overlay_pnl_by_year": {str(year): round(value, 2) for year, value in sorted(pnl_by_year.items())},
        }
        rows.append(row)
    return rows


def main() -> None:
    parser = argparse.ArgumentParser(description="Probe non-parking idle-capital ideas")
    subparsers = parser.add_subparsers(dest="command", required=True)

    overnight_parser = subparsers.add_parser("overnight", help="Idle-cash overnight drift overlay")
    overnight_parser.add_argument("--config", action="append", required=True)
    overnight_parser.add_argument("--start", required=True)
    overnight_parser.add_argument("--end", required=True)
    overnight_parser.add_argument("--capital", type=float, default=10_000.0)
    overnight_parser.add_argument("--symbol", choices=("SPY", "QQQ"), default="SPY")
    overnight_parser.add_argument("--parking-preset", default=None)
    overnight_parser.add_argument("--min-cash-pct", type=float, default=0.0)
    overnight_parser.add_argument("--require-no-open-positions", action="store_true")
    overnight_parser.add_argument("--json", action="store_true")

    pairs_parser = subparsers.add_parser("pairs", help="Simple mean-reversion pair sanity check")
    pairs_parser.add_argument("--config", required=True)
    pairs_parser.add_argument("--start", required=True)
    pairs_parser.add_argument("--end", required=True)
    pairs_parser.add_argument("--capital", type=float, default=10_000.0)
    pairs_parser.add_argument(
        "--pair",
        action="append",
        required=True,
        help="Pair in SYMBOL_A/SYMBOL_B form. Repeatable.",
    )
    pairs_parser.add_argument("--lookback", type=int, default=60)
    pairs_parser.add_argument("--enter-z", type=float, default=2.0)
    pairs_parser.add_argument("--exit-z", type=float, default=0.5)
    pairs_parser.add_argument("--max-hold", type=int, default=20)
    pairs_parser.add_argument("--grid", action="store_true")
    pairs_parser.add_argument("--json", action="store_true")

    micro_parser = subparsers.add_parser(
        "micro-event",
        help="Test small capital-bucket event sleeves using existing strategy engines",
    )
    micro_parser.add_argument("--config", required=True)
    micro_parser.add_argument("--start", required=True)
    micro_parser.add_argument("--end", required=True)
    micro_parser.add_argument("--capital", type=float, default=10_000.0)
    micro_parser.add_argument(
        "--engine-id",
        action="append",
        required=True,
        help="Existing engine_id to clone or bucket. Repeatable.",
    )
    micro_parser.add_argument("--mode", choices=("clone", "bucket"), default="clone")
    micro_parser.add_argument("--bucket-id", default="micro_event")
    micro_parser.add_argument("--bucket-allocation-pct", type=float, required=True)
    micro_parser.add_argument("--per-trade-risk-pct", type=float, default=None)
    micro_parser.add_argument("--selection-priority", type=int, default=-1)
    micro_parser.add_argument("--parking-preset", default=None)
    micro_parser.add_argument("--json", action="store_true")

    weekday_parser = subparsers.add_parser(
        "weekday-collateral",
        help="Intraday weekday sleeve funded by buying power or reserved cash",
    )
    weekday_parser.add_argument("--config", action="append", required=True)
    weekday_parser.add_argument("--start", required=True)
    weekday_parser.add_argument("--end", required=True)
    weekday_parser.add_argument("--capital", type=float, default=10_000.0)
    weekday_parser.add_argument("--parking-preset", default="qqqm_low_dd")
    weekday_parser.add_argument("--symbol", choices=("SPY", "QQQ"), default="SPY")
    weekday_parser.add_argument("--weekday", type=int, choices=range(5), default=0)
    weekday_parser.add_argument(
        "--filter",
        choices=("none", "trend50", "mom20", "vol15", "trend50_vol15"),
        default="none",
    )
    weekday_parser.add_argument(
        "--capital-source",
        choices=("cash_available", "reserve_pct"),
        default="cash_available",
    )
    weekday_parser.add_argument("--reserve-pct", type=float, default=0.0)
    weekday_parser.add_argument("--json", action="store_true")

    args = parser.parse_args()
    configure_logging("WARNING")

    if args.command == "overnight":
        rows = _run_overnight_probe(args)
    elif args.command == "pairs":
        rows = _run_pairs_probe(args)
    elif args.command == "weekday-collateral":
        rows = _run_weekday_collateral_probe(args)
    else:
        rows = _run_micro_event_probe(args)

    if args.json:
        print(json.dumps(rows, indent=2))
    else:
        for row in rows:
            print(row)


if __name__ == "__main__":
    main()