fithia2/apps/intraday_bt/scripts/diag_gapdown_short.py

"""
Gap-Down ORB Short Diagnostic (Option 3)

Hypothesis: stocks that GAP DOWN ≥2% and have a bearish ORB (5-min ORB candle close < open)
continue lower and offer a short-side edge that is regime-orthogonal to V23.

Key difference vs Phase 3 (hypergap_failure_v1):
  - Phase 3 shorted GAP-UP stocks that FAILED: adverse R/R (rockets up when wrong)
  - This shorts GAP-DOWN stocks that CONTINUE lower: more symmetric R/R
    (when wrong: slow recovery, not a rocket; when right: fills gap to prev close)

Setup:
  - Universe: midlarge tickers with daily cache (for gap computation)
  - Gap: today_open/prev_close - 1 ≤ -0.02 (gap down ≥ 2%)
  - ORB filter: 9:30-9:35 ET candle close < open (bearish ORB)
  - Entry: short below ORB low (when price breaks below ORB low)
  - Stop: above ORB high (ATR-based stop as alternative if ORB range too wide)
  - Target 1: prev_close (gap fill) — partial exit
  - Target 2: ATR * 1.5 below entry
  - Exit: EOD (15:55 ET) if no target hit

Quality filters (same as V23):
  - min_price: 10.0
  - min_avg_dollar_volume: 25M (proxy from daily bar volume * close)
  - min_rvol: 1.5 approx (first 5-min vol vs avg bar vol)
  - max_gap_pct: -0.10 (don't short extreme gap-downs — catastrophic reversal risk)

Gates:
  - WR ≥ 42% (shorts can work at lower WR if R/R > 1.5)
  - avg_win / avg_loss ≥ 1.2 (need favorable R/R for shorts)
  - Pearson corr with V23 daily PnL ≤ 0.00 (should be negative or near-zero)
"""
from __future__ import annotations

import json
import math
import os
import sys
from pathlib import Path

import numpy as np
import pandas as pd
import yaml

# ── Config ─────────────────────────────────────────────────────────────────

CACHE_DIR = "data/cache/intraday"
DAILY_CACHE_DIR = "data/cache/daily"
UNIVERSE_FILE = "configs/symbols_midlarge_snapshot_exact.yaml"
V23_BASELINE_RUN = "runs/intraday_orb/intraday_20260420_205136_5597b16d.json"

MIN_PRICE = 10.0
MIN_GAP_DOWN = -0.02       # gap ≤ -2%
MAX_GAP_DOWN = -0.10       # don't short extreme gaps (≤ -10%)
MIN_AVG_DOLLAR_VOL = 25_000_000
MIN_RVOL = 1.5
ATR_STOP_MULT = 1.0        # stop above ORB high (or ATR above entry if ORB range > ATR)
RISK_PER_TRADE = 500.0     # $500 = 5% of $10k
MAX_SIMULTANEOUS = 3
EXIT_HOUR = 15
EXIT_MIN = 55


def load_universe() -> list[str]:
    with open(UNIVERSE_FILE) as f:
        data = yaml.safe_load(f)
    return data.get("symbols", data) if isinstance(data, dict) else data


def load_daily_cache(ticker: str) -> pd.DataFrame | None:
    path = f"{DAILY_CACHE_DIR}/{ticker}.parquet"
    if not os.path.exists(path):
        return None
    try:
        return pd.read_parquet(path)
    except Exception:
        return None


def load_bars(ticker: str, date: str) -> pd.DataFrame | None:
    path = f"{CACHE_DIR}/{ticker}/{date}.parquet"
    if not os.path.exists(path):
        return None
    try:
        df = pd.read_parquet(path)
        if len(df) < 10:
            return None
        df["ts"] = pd.to_datetime(df["timestamp"]).dt.tz_convert("US/Eastern")
        df["hour"] = df["ts"].dt.hour
        df["minute"] = df["ts"].dt.minute
        df = df[(df["hour"] >= 9) & (df["hour"] < 16)].copy()
        df = df.sort_values("ts").reset_index(drop=True)
        return df
    except Exception:
        return None


def approximate_atr(df: pd.DataFrame) -> float:
    ranges = df["high"] - df["low"]
    return ranges.mean() * math.sqrt(78)


def compute_rvol(bars: pd.DataFrame, avg_daily_vol: float) -> float:
    """Approx rvol: first 5-min volume / (avg_daily_vol / 78)."""
    if avg_daily_vol <= 0 or len(bars) == 0:
        return 0.0
    first_vol = bars.iloc[0]["volume"]
    return first_vol / (avg_daily_vol / 78)


def simulate_day(date: str, tickers: list[str],
                 daily_data: dict[str, pd.DataFrame]) -> dict:
    """Run gap-down short simulation for one day."""
    day_trades: list[dict] = []
    open_positions: int = 0

    # Pre-filter candidates using daily data
    candidates = []
    for ticker in tickers:
        if ticker not in daily_data:
            continue
        df = daily_data[ticker]
        idx = df.index[df["date"] == date].tolist()
        if not idx:
            continue
        row_idx = idx[0]
        if row_idx == 0:
            continue  # no prev day
        today = df.iloc[row_idx]
        prev = df.iloc[row_idx - 1]
        if prev["close"] <= 0:
            continue

        gap = (today["open"] - prev["close"]) / prev["close"]
        if gap > MIN_GAP_DOWN or gap < MAX_GAP_DOWN:
            continue

        # Dollar volume filter (proxy)
        avg_dvol = df["close"].iloc[max(0, row_idx - 20):row_idx].mean() * df["volume"].iloc[max(0, row_idx - 20):row_idx].mean()
        if avg_dvol < MIN_AVG_DOLLAR_VOL:
            continue

        candidates.append((ticker, gap, prev["close"], today["open"]))

    # Sort by gap magnitude (largest gap-down first)
    candidates.sort(key=lambda x: x[1])

    for ticker, gap, prev_close, today_open in candidates:
        if open_positions >= MAX_SIMULTANEOUS:
            break

        bars = load_bars(ticker, date)
        if bars is None or len(bars) < 10:
            continue

        # Price check
        if bars.iloc[0]["open"] < MIN_PRICE:
            continue

        # ORB bar: first bar at/after 9:30 ET
        orb_bars = bars[(bars["hour"] == 9) & (bars["minute"] >= 30) & (bars["minute"] < 35)]
        if len(orb_bars) == 0:
            continue
        orb = orb_bars.iloc[0]

        # Bearish ORB: close < open
        if orb["close"] >= orb["open"]:
            continue

        # RVOL check
        avg_daily = daily_data[ticker]["volume"].mean() if ticker in daily_data else 0
        rvol = compute_rvol(bars, avg_daily)
        if rvol < MIN_RVOL:
            continue

        # ATR for stop sizing
        atr = approximate_atr(bars)
        if atr <= 0:
            continue

        orb_low = orb["low"]
        orb_high = orb["high"]
        entry_trigger = orb_low  # short below ORB low

        # Stop: above ORB high (capped at ATR_STOP_MULT * ATR above entry_trigger)
        stop_distance = orb_high - entry_trigger
        max_stop_dist = ATR_STOP_MULT * atr
        stop_price = entry_trigger + min(stop_distance, max_stop_dist)

        if stop_price <= entry_trigger:
            continue
        shares = RISK_PER_TRADE / (stop_price - entry_trigger)
        if shares <= 0 or shares * entry_trigger > 50000:
            continue

        # Profit target: ATR * 1.5 below entry (stock continues falling)
        # Gap fill recovery (additional stop): if stock recovers above prev_close → exit at loss
        gap_recovery_stop = prev_close  # stock fully recovered = worst case stop
        atr_target_dist = 1.5 * atr    # short profit target = entry - 1.5 ATR

        # Scan intraday bars after ORB for entry trigger
        post_orb = bars[bars.index > orb_bars.index[0]]
        entry_price = None
        exit_price = None
        exit_reason = "no_entry"
        pnl = 0.0
        profit_target_price: float = 0.0

        for _, bar in post_orb.iterrows():
            if entry_price is None:
                # Look for short entry: bar low breaks below ORB low
                if bar["low"] <= entry_trigger:
                    entry_price = min(entry_trigger, bar["open"])
                    profit_target_price = entry_price - atr_target_dist
                    # Check stop immediately (gap-through entry on same bar)
                    if bar["high"] >= stop_price:
                        exit_price = stop_price
                        exit_reason = "stop"
                        pnl = (entry_price - exit_price) * shares
                        break
                    continue
            else:
                # Position open — check stop first, then target, then EOD
                if bar["hour"] >= EXIT_HOUR and bar["minute"] >= EXIT_MIN:
                    exit_price = bar["close"]
                    exit_reason = "eod"
                    break
                # Stop: above ORB high, or stock fully recovers gap
                if bar["high"] >= stop_price or bar["high"] >= gap_recovery_stop:
                    exit_price = max(stop_price, bar["open"])
                    exit_reason = "stop"
                    break
                # Profit target: price falls ATR * 1.5 below entry
                if profit_target_price > 0 and bar["low"] <= profit_target_price:
                    exit_price = max(profit_target_price, bar["open"])
                    exit_reason = "target"
                    break

        if entry_price is None:
            continue  # no entry triggered

        if exit_price is None:
            # Last bar
            exit_price = bars.iloc[-1]["close"]
            exit_reason = "eod"

        pnl = (entry_price - exit_price) * shares  # short pnl

        day_trades.append({
            "ticker": ticker,
            "date": date,
            "gap": gap,
            "entry": entry_price,
            "exit": exit_price,
            "stop": stop_price,
            "target": prev_close,
            "shares": shares,
            "pnl": pnl,
            "win": pnl > 0,
            "exit_reason": exit_reason,
        })
        open_positions += 1

    day_pnl = sum(t["pnl"] for t in day_trades)
    return {"date": date, "trades": day_trades, "day_pnl": day_pnl}


def main() -> None:
    print("=== Gap-Down ORB Short Diagnostic ===\n")

    with open(V23_BASELINE_RUN) as f:
        v23_data = json.load(f)
    v23_dates = [d["date"] for d in v23_data["daily_summary"]]
    v23_pnl = {d["date"]: d["daily_pnl"] for d in v23_data["daily_summary"]}

    universe = load_universe()
    print(f"Universe: {len(universe)} tickers")
    print(f"Loading daily caches...", flush=True)

    daily_data: dict[str, pd.DataFrame] = {}
    for ticker in universe:
        df = load_daily_cache(ticker)
        if df is not None and len(df) >= 2:
            daily_data[ticker] = df

    print(f"Daily cache loaded: {len(daily_data)} tickers")
    print(f"V23 window: {v23_dates[0]} → {v23_dates[-1]} ({len(v23_dates)} days)\n")

    all_results = []
    for i, date in enumerate(v23_dates):
        result = simulate_day(date, universe, daily_data)
        all_results.append(result)
        if (i + 1) % 20 == 0:
            print(f"  {i+1}/{len(v23_dates)} days processed...", flush=True)

    # Aggregate
    all_trades = [t for r in all_results for t in r["trades"]]
    total_trades = len(all_trades)
    wins = sum(1 for t in all_trades if t["win"])
    win_rate = wins / total_trades if total_trades else 0.0
    avg_win = float(np.mean([t["pnl"] for t in all_trades if t["pnl"] > 0])) if wins > 0 else 0.0
    avg_loss = float(np.mean([abs(t["pnl"]) for t in all_trades if t["pnl"] <= 0])) if total_trades - wins > 0 else 0.0
    total_pnl = sum(t["pnl"] for t in all_trades)
    exit_reasons = {}
    for t in all_trades:
        exit_reasons[t["exit_reason"]] = exit_reasons.get(t["exit_reason"], 0) + 1

    # Correlation with V23
    gd_daily = {r["date"]: r["day_pnl"] for r in all_results}
    gd_series = [gd_daily.get(d, 0.0) for d in v23_dates]
    v23_series = [v23_pnl.get(d, 0.0) for d in v23_dates]
    corr = float(np.corrcoef(gd_series, v23_series)[0, 1]) if len(v23_dates) > 1 else 0.0

    trade_days = sum(1 for r in all_results if r["trades"])

    print("=" * 55)
    print(f"  Total trades:     {total_trades}")
    print(f"  Trade days:       {trade_days} / {len(v23_dates)}")
    print(f"  Win rate:         {win_rate*100:.1f}%  (gate: ≥42%)")
    print(f"  Avg win:          ${avg_win:.2f}")
    print(f"  Avg loss:         ${avg_loss:.2f}")
    if avg_loss > 0:
        print(f"  Win/Loss ratio:   {avg_win/avg_loss:.2f}  (gate: ≥1.2)")
    print(f"  Total PnL:        ${total_pnl:+.2f}")
    print(f"  Corr vs V23:      {corr:.3f}  (gate: ≤0.00)")
    print(f"  Exit breakdown:   {exit_reasons}")
    print("=" * 55)

    g1 = win_rate >= 0.42
    g2 = (avg_win / avg_loss >= 1.2) if avg_loss > 0 else False
    g3 = corr <= 0.00
    g4 = total_trades >= 30
    print(f"\nGate G1 (WR ≥ 42%):        {'PASS' if g1 else 'FAIL'}")
    print(f"Gate G2 (W/L ≥ 1.2):       {'PASS' if g2 else 'FAIL'}")
    print(f"Gate G3 (corr ≤ 0.00):     {'PASS' if g3 else 'FAIL'}")
    print(f"Gate G4 (trades ≥ 30):     {'PASS' if g4 else 'FAIL'}")
    verdict = "PROCEED TO ENGINE BUILD" if (g1 and g2 and g3 and g4) else "ABORT — edge not confirmed"
    print(f"\nVERDICT: {verdict}")

    out = {
        "total_trades": total_trades,
        "trade_days": trade_days,
        "win_rate": win_rate,
        "avg_win": avg_win,
        "avg_loss": avg_loss,
        "total_pnl": total_pnl,
        "corr_v23": corr,
        "exit_reasons": exit_reasons,
        "trades": all_trades,
        "daily_pnl": [{"date": r["date"], "pnl": r["day_pnl"]} for r in all_results],
    }
    out_path = "runs/intraday_orb/diag_gapdown_short.json"
    with open(out_path, "w") as f:
        json.dump(out, f, indent=2, default=str)
    print(f"\nResults saved to {out_path}")


if __name__ == "__main__":
    sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../..")))
    main()