fithia2/apps/intraday_bt/scripts/p22_pre_universe_shift.py

"""P22-pre: Universe-shift gainers_leader proxy diagnostic.

V49 trades on a $10+ price / $25M+ ADV slice. This probe checks whether the
gainers_leader pattern (top %-gap-up names → buy at open, exit at close)
produces positive expectancy on OTHER price × ADV slices of the snapshot.

The pinned daily snapshot orb_daily_v46_20260423 has 6540 tickers — a
superset of V49's 3408-ticker screener universe. We bin all tickers by
price × ADV, then for each bin × each day, pick top-5 by gap (gap >= +2%)
and average open-to-close return. That's the gainers_leader 'long the
biggest gappers at open' expectancy proxy.

KILL gate: if no slice has materially stronger expectancy than the baseline
($10+/$25M+) and no slice has positive expectancy in a previously-untouched
zone, autonomous probe space is exhausted — append V49 FINAL to lineage.
"""
from __future__ import annotations

from pathlib import Path

import pandas as pd

DAILY_DIR = Path("data/cache/daily_snapshots/orb_daily_v46_20260423")
WINDOW_START = pd.Timestamp("2025-07-09")
WINDOW_END = pd.Timestamp("2026-04-23")
GAP_THR = 0.02
TOP_N = 5

PRICE_BINS = [
    ("$2-$5",     2.0,    5.0),
    ("$5-$10",    5.0,    10.0),
    ("$10-$25",   10.0,   25.0),
    ("$25-$100",  25.0,   100.0),
    ("$100+",     100.0,  1e9),
]
ADV_BINS = [
    ("$0.5M-$5M",  0.5e6,  5e6),
    ("$5M-$25M",   5e6,    25e6),
    ("$25M-$100M", 25e6,   100e6),
    ("$100M+",     100e6,  1e12),
]


def main() -> None:
    files = sorted(DAILY_DIR.glob("*.parquet"))
    print(f"Loading {len(files)} ticker daily snapshots...")
    rows = []
    skipped = 0
    for fp in files:
        try:
            df = pd.read_parquet(fp)
        except Exception:
            skipped += 1
            continue
        if len(df) < 25:
            skipped += 1
            continue
        sym = fp.stem
        df = df.sort_values("date").reset_index(drop=True)
        df["date"] = pd.to_datetime(df["date"])
        df["prev_close"] = df["close"].shift(1)
        df["gap"] = (df["open"] / df["prev_close"]) - 1.0
        df["oc_ret"] = (df["close"] / df["open"]) - 1.0
        df["dollar_vol"] = df["close"] * df["volume"]
        df["adv20"] = df["dollar_vol"].rolling(20).mean()
        # filter to window and to rows with all features
        sub = df[
            (df["date"] >= WINDOW_START)
            & (df["date"] <= WINDOW_END)
            & df["gap"].notna()
            & df["adv20"].notna()
            & (df["open"] > 0)
            & (df["close"] > 0)
        ].copy()
        if len(sub):
            sub["ticker"] = sym
            sub["price"] = sub["close"]  # use close as price band proxy
            rows.append(sub[["ticker", "date", "gap", "oc_ret", "adv20", "price"]])

    if not rows:
        print("No data.")
        return
    big = pd.concat(rows, ignore_index=True)
    print(f"Total ticker-days: {len(big):,} (skipped {skipped} files)")
    print(f"Unique tickers: {big['ticker'].nunique():,}")
    print(f"Days in window: {big['date'].nunique()}")
    print()

    # Per-bin gainers_leader proxy
    print(f"=== Gainers_leader proxy: top-{TOP_N} by gap (gap >= +{GAP_THR*100:.0f}%) per day ===")
    print(f"Trade: long at open, exit at close. Universe = each (price × ADV) bin.")
    print()
    print(f"{'price_bin':>10} | {'adv_bin':>13} | {'tickers':>7} | {'days':>5} | {'n_trades':>8} | {'gap_avg%':>9} | {'WR%':>6} | {'avg_win%':>9} | {'avg_loss%':>10} | {'expectancy%':>12} | {'cum%':>10}")
    print("-" * 145)

    summary = []
    for pname, plo, phi in PRICE_BINS:
        for aname, alo, ahi in ADV_BINS:
            mask = (
                (big["price"] >= plo)
                & (big["price"] < phi)
                & (big["adv20"] >= alo)
                & (big["adv20"] < ahi)
                & (big["gap"] >= GAP_THR)
            )
            slice_df = big[mask]
            n_tickers = slice_df["ticker"].nunique()
            n_days = slice_df["date"].nunique()
            if n_days == 0:
                continue
            # For each day, pick top-N by gap
            picks = (
                slice_df.sort_values(["date", "gap"], ascending=[True, False])
                .groupby("date")
                .head(TOP_N)
            )
            n = len(picks)
            if n < 30:
                continue
            wins = picks["oc_ret"][picks["oc_ret"] > 0]
            losses = picks["oc_ret"][picks["oc_ret"] < 0]
            wr = len(wins) / n
            avg_win = wins.mean() if len(wins) else 0
            avg_loss = -losses.mean() if len(losses) else 0
            expectancy = picks["oc_ret"].mean()
            gap_avg = picks["gap"].mean()
            cum = picks["oc_ret"].sum()
            summary.append({
                "price_bin": pname,
                "adv_bin": aname,
                "tickers": n_tickers,
                "days": n_days,
                "n_trades": n,
                "gap_avg": gap_avg,
                "wr": wr,
                "avg_win": avg_win,
                "avg_loss": avg_loss,
                "expectancy": expectancy,
                "cum": cum,
            })
            print(f"{pname:>10} | {aname:>13} | {n_tickers:>7} | {n_days:>5} | {n:>8} | {gap_avg*100:>8.2f} | {wr*100:>5.1f} | {avg_win*100:>8.3f} | {avg_loss*100:>9.3f} | {expectancy*100:>+11.4f} | {cum*100:>+9.2f}")

    if not summary:
        print("No bins met n>=30 threshold.")
        return

    print()
    print(f"=== V49 baseline reference: $10+ price, $25M+ ADV ===")
    print(f"V49 actually trades 166 trades over 200d with +155.58% return and 50%+ WR via")
    print(f"the full gainers_leader engine (event_catalyst, obv, ORB breakout, etc.).")
    print(f"This proxy uses ONLY 'long top-N gappers at open, exit at close' — far weaker.")
    print(f"Look for slices where the *proxy* itself shows materially better expectancy")
    print(f"than the 'comparable' bins (price=$10-$25 / adv=$25M+) — those are the")
    print(f"directional candidates worth engine work.")
    print()

    # Highlight best/worst slices
    sdf = pd.DataFrame(summary).sort_values("expectancy", ascending=False)
    print("=== Top 5 slices by expectancy ===")
    print(sdf.head(5).to_string(index=False, formatters={
        "gap_avg": "{:+.2%}".format,
        "wr": "{:.1%}".format,
        "avg_win": "{:.2%}".format,
        "avg_loss": "{:.2%}".format,
        "expectancy": "{:+.4%}".format,
        "cum": "{:+.2%}".format,
    }))
    print()
    print("=== Bottom 5 slices by expectancy ===")
    print(sdf.tail(5).to_string(index=False, formatters={
        "gap_avg": "{:+.2%}".format,
        "wr": "{:.1%}".format,
        "avg_win": "{:.2%}".format,
        "avg_loss": "{:.2%}".format,
        "expectancy": "{:+.4%}".format,
        "cum": "{:+.2%}".format,
    }))

    # Comparison: V49-equivalent slices vs alternative slices
    v49_eq = sdf[(sdf["price_bin"].isin(["$10-$25", "$25-$100", "$100+"])) & (sdf["adv_bin"].isin(["$25M-$100M", "$100M+"]))]
    if len(v49_eq):
        print()
        print(f"=== V49-equivalent slices (price >= $10, ADV >= $25M), n_bins={len(v49_eq)} ===")
        print(f"Avg expectancy: {v49_eq['expectancy'].mean()*100:+.4f}%")
        print(f"Avg WR: {v49_eq['wr'].mean()*100:.2f}%")
    alt = sdf[~sdf.index.isin(v49_eq.index)]
    if len(alt):
        print()
        print(f"=== Alternative slices (NEW universe candidates), n_bins={len(alt)} ===")
        print(f"Avg expectancy: {alt['expectancy'].mean()*100:+.4f}%")
        print(f"Avg WR: {alt['wr'].mean()*100:.2f}%")
        better_alt = alt[alt["expectancy"] > v49_eq["expectancy"].mean() if len(v49_eq) else 0]
        print(f"Alt slices that beat V49-equivalent avg expectancy: {len(better_alt)}/{len(alt)}")


if __name__ == "__main__":
    main()