fithia2/apps/intraday_bt/scripts/diag_orb_remaining.py

"""
V38 Diagnostic: Remaining unexplored daily feature axes

Tests 4 features not yet explored against V24 trade set (400d, n=304):

  1. dollar_vol_trend  : avg_dollar_vol_10d / avg_dollar_vol_30d — is dollar volume
                         ACCELERATING? If yes, recent activity surge above baseline.
  2. sleep_streak      : consecutive prior days with |daily_return| < 1.5% —
                         "coiled spring" hypothesis: long quiet period before explosive gap.
  3. prior_day_return  : yesterday's close vs. day-before close (1-day momentum/extension check).
                         Hypothesis: stocks that were flat/down yesterday have better ORB
                         follow-through (less extension) than stocks already up yesterday.
  4. vol_trend_ratio   : avg_volume_5d / avg_volume_20d — is volume accelerating (short-term)?
"""
from __future__ import annotations

import concurrent.futures
import datetime as dt
import json
import os
import sys
from pathlib import Path

import pyarrow.parquet as pq

sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../..")))

from zoneinfo import ZoneInfo

_ET = ZoneInfo("America/New_York")
_MKT_OPEN = dt.time(9, 30)
_MKT_CLOSE = dt.time(16, 0)

INTRADAY_CACHE_DIR = "data/cache/intraday"
V24_400D_RUN = "runs/intraday_orb/intraday_20260422_011012_06f59ede.json"


def _parse_ts(ts_raw: object) -> dt.datetime:
    s = str(ts_raw)
    if s.endswith("Z"):
        s = s[:-1] + "+00:00"
    return dt.datetime.fromisoformat(s).astimezone(_ET)


def _build_daily_bar(path: Path, date: str) -> dict | None:
    try:
        table = pq.read_table(str(path))
        rows = table.to_pydict()
    except Exception:
        return None
    opens, closes, vols = [], [], []
    for i, ts_raw in enumerate(rows.get("timestamp", [])):
        try:
            ts = _parse_ts(ts_raw)
        except Exception:
            continue
        if _MKT_OPEN <= ts.time() < _MKT_CLOSE:
            opens.append(float(rows["open"][i] or 0))
            closes.append(float(rows["close"][i] or 0))
            vols.append(float(rows["volume"][i] or 0))
    if not closes or closes[-1] <= 0:
        return None
    close = closes[-1]
    vol = sum(vols)
    return {"date": date, "open": opens[0] if opens else 0, "close": close,
            "volume": vol, "dollar_vol": close * vol}


def load_ticker_bars(ticker: str, all_dates: list[str]) -> list[dict]:
    root = Path(INTRADAY_CACHE_DIR) / ticker
    if not root.is_dir():
        return []
    bars = []
    for date in all_dates:
        p = root / f"{date}.parquet"
        if not p.exists():
            continue
        bar = _build_daily_bar(p, date)
        if bar and bar["close"] > 0:
            bars.append(bar)
    return sorted(bars, key=lambda b: b["date"])


def _avg(vals: list[float]) -> float:
    return sum(vals) / len(vals) if vals else 0.0


def compute_dollar_vol_trend(prev_bars: list[dict]) -> float | None:
    if len(prev_bars) < 30:
        return None
    dv_5 = _avg([b["dollar_vol"] for b in prev_bars[-10:]])
    dv_30 = _avg([b["dollar_vol"] for b in prev_bars[-30:]])
    if dv_30 <= 0:
        return None
    return dv_5 / dv_30


def compute_sleep_streak(prev_bars: list[dict], threshold: float = 0.015) -> float | None:
    """Count consecutive prior days with |return| < threshold."""
    if len(prev_bars) < 2:
        return None
    streak = 0
    for i in range(len(prev_bars) - 1, 0, -1):
        c = prev_bars[i]["close"]
        pc = prev_bars[i - 1]["close"]
        if pc <= 0:
            break
        ret = abs((c - pc) / pc)
        if ret < threshold:
            streak += 1
        else:
            break
    return float(streak)


def compute_prior_day_return(prev_bars: list[dict]) -> float | None:
    if len(prev_bars) < 2:
        return None
    c = prev_bars[-1]["close"]
    pc = prev_bars[-2]["close"]
    if pc <= 0:
        return None
    return (c - pc) / pc


def compute_vol_trend_ratio(prev_bars: list[dict]) -> float | None:
    if len(prev_bars) < 20:
        return None
    v5 = _avg([b["volume"] for b in prev_bars[-5:]])
    v20 = _avg([b["volume"] for b in prev_bars[-20:]])
    if v20 <= 0:
        return None
    return v5 / v20


def pearson(xs: list[float], ys: list[float]) -> float | None:
    n = len(xs)
    if n < 2:
        return None
    xm, ym = sum(xs) / n, sum(ys) / n
    num = sum((xs[i] - xm) * (ys[i] - ym) for i in range(n))
    dx = sum((x - xm) ** 2 for x in xs) ** 0.5
    dy = sum((y - ym) ** 2 for y in ys) ** 0.5
    if dx <= 0 or dy <= 0:
        return None
    return num / (dx * dy)


def tercile_stats(vals: list[float], rs: list[float]) -> dict:
    if len(vals) < 9:
        return {}
    pairs = sorted(zip(vals, rs), key=lambda p: p[0])
    n = len(pairs)
    t = n // 3
    def stats(sub):
        ys = [p[1] for p in sub]
        return {"n": len(ys), "wr": sum(1 for y in ys if y > 0) / len(ys), "avg_r": sum(ys) / len(ys)}
    return {"low": stats(pairs[:t]), "mid": stats(pairs[t:2*t]), "high": stats(pairs[2*t:])}


def report_feature(label: str, vals: list[float], rs: list[float]) -> None:
    n = len(vals)
    p = pearson(vals, rs)
    ts = tercile_stats(vals, rs)
    if not ts or p is None:
        print(f"  {label}: n={n}, insufficient data")
        return
    low, mid, high = ts["low"], ts["mid"], ts["high"]
    avg_r_gap = abs(high["avg_r"] - low["avg_r"])
    wr_gap = abs(high["wr"] - low["wr"])
    g1 = abs(p) >= 0.07 and n >= 120
    g2 = avg_r_gap >= 0.30
    g3 = wr_gap >= 0.05
    print(f"\n  [{label}]  n={n}  Pearson={p:+.3f}")
    print(f"    G1: {'PASS' if g1 else 'FAIL'} (|{abs(p):.3f}| {'≥' if abs(p)>=0.07 else '<'} 0.07, n={n})")
    print(f"    G2: {'PASS' if g2 else 'FAIL'} (avg_R gap = {avg_r_gap:.3f}R  [threshold 0.30R])")
    print(f"    G3: {'PASS' if g3 else 'FAIL'} (WR gap = {wr_gap*100:.1f}pp  [threshold 5pp])")
    print(f"    Bottom tercile: WR {low['wr']*100:.1f}%  avg_R {low['avg_r']:+.3f}  n={low['n']}")
    print(f"    Middle tercile: WR {mid['wr']*100:.1f}%  avg_R {mid['avg_r']:+.3f}  n={mid['n']}")
    print(f"    Top tercile:    WR {high['wr']*100:.1f}%  avg_R {high['avg_r']:+.3f}  n={high['n']}")
    print(f"    → {'ALL GATES PASS ✓' if (g1 and g2 and g3) else 'FAIL'}")


def main() -> None:
    print("=== V38 Remaining Daily Features Diagnostic ===\n")

    with open(V24_400D_RUN) as f:
        run_data = json.load(f)
    trades = run_data.get("trades", [])
    m = run_data.get("metrics", {})
    print(f"Loaded V24 400d run: {m.get('total_trades')} trades, "
          f"{m.get('start_date')} → {m.get('end_date')}")
    print(f"Return: {m.get('total_return_pct', 0)*100:.2f}%  DD: {m.get('max_drawdown_pct', 0)*100:.2f}%  Sharpe: {m.get('sharpe_ratio', 0):.3f}\n")

    trade_records = [
        {"ticker": t["ticker"], "date": t["date"][:10],
         "r_multiple": float(t["r_multiple_at_exit"])}
        for t in trades
        if t.get("r_multiple_at_exit") is not None
    ]
    print(f"Trades with r_multiple: {len(trade_records)}")

    tickers_needed = sorted(set(r["ticker"] for r in trade_records))
    min_date = min(r["date"] for r in trade_records)
    max_date = max(r["date"] for r in trade_records)
    start_cal = (dt.date.fromisoformat(min_date) - dt.timedelta(days=120)).isoformat()
    all_dates = []
    d = dt.date.fromisoformat(start_cal)
    end_d = dt.date.fromisoformat(max_date)
    while d <= end_d:
        all_dates.append(d.isoformat())
        d += dt.timedelta(days=1)

    print(f"Loading bars for {len(tickers_needed)} tickers...")
    ticker_bars: dict[str, list[dict]] = {}
    with concurrent.futures.ThreadPoolExecutor(max_workers=8) as ex:
        def _load(ticker: str) -> tuple[str, list[dict]]:
            return ticker, load_ticker_bars(ticker, all_dates)
        for ticker, bars in ex.map(_load, tickers_needed):
            if bars:
                ticker_bars[ticker] = bars
    print(f"Loaded: {len(ticker_bars)} / {len(tickers_needed)} tickers\n")

    dv_trend_all, sleep_all, prior_ret_all, vol_trend_all, r_mults_all = [], [], [], [], []

    for rec in trade_records:
        ticker = rec["ticker"]
        date = rec["date"]
        r = rec["r_multiple"]
        bars = ticker_bars.get(ticker, [])
        prev = [b for b in bars if b["date"] < date]

        dv = compute_dollar_vol_trend(prev)
        sl = compute_sleep_streak(prev)
        pr = compute_prior_day_return(prev)
        vt = compute_vol_trend_ratio(prev)

        if dv is None or sl is None or pr is None or vt is None:
            continue

        dv_trend_all.append(dv)
        sleep_all.append(sl)
        prior_ret_all.append(pr)
        vol_trend_all.append(vt)
        r_mults_all.append(r)

    print(f"Valid trades (all 4 features): {len(r_mults_all)} / {len(trade_records)}")

    print("\n" + "=" * 60)
    print("GATE RESULTS (G1: |P|≥0.07 & n≥120; G2: avg_R≥0.30R; G3: WR≥5pp)")

    report_feature("dollar_vol_trend (dvol10d/dvol30d)", dv_trend_all, r_mults_all)
    report_feature("sleep_streak (consec. low-vol days)", sleep_all, r_mults_all)
    report_feature("prior_day_return", prior_ret_all, r_mults_all)
    report_feature("vol_trend_ratio (vol5d/vol20d)", vol_trend_all, r_mults_all)

    print("\n=== Benchmark ===")
    print("OBV-slope 20d: Pearson=+0.235, G2=0.394R — only passing signal.")
    print("Any G2 ≥ 0.30R here → advance to wiring. Otherwise → V24 is terminal for daily OHLCV axis.")


if __name__ == "__main__":
    main()