"""
V25 Short-Volume Feature Diagnostic (on V46 400d trade set).

Uses DB short_sale_daily table (populated by backfill_finra_short_volume_cdn.py).
Tests three short-ratio features on the 400d V46 simulation trade set.

STOP CONDITION: If no feature clears G2≥0.30R, V46 is terminal for this session.
"""
from __future__ import annotations

import asyncio
import concurrent.futures
import datetime as dt
import json
import math
import os
import statistics
import sys
from pathlib import Path

import asyncpg
import yaml
from zoneinfo import ZoneInfo

sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../..")))

from libs.common.config import get_settings
from libs.common.time_utils import trading_days_between
from libs.intraday.domain import ORBStrategyParams
from libs.intraday.features import compute_obv_slope_approx, enrich_daily_bars
from libs.intraday.orb_simulator import ORBSimulationState, run_orb_simulation_with_state
from libs.intraday.screener import orb_pre_screen_candidates

# Use V46 config (PEAD disabled at simulation level — no event prefetch in diagnostic)
V46_CONFIG = "configs/intraday/strategies/orb_gainers_v46_prior_event.yaml"
UNIVERSE_FILE = "configs/symbols_midlarge_snapshot_exact.yaml"
INTRADAY_CACHE_DIR = "data/cache/intraday"
LOOKBACK_DAYS = 400  # 400d for statistical power (≥300 trades)
FEATURE_LOOKBACK_TRADING = 25

_ET = ZoneInfo("America/New_York")
_MKT_OPEN = dt.time(9, 30)
_MKT_CLOSE = dt.time(16, 0)


# ── DB Short Volume Loader ───────────────────────────────────────────────────

async def _load_short_vol_from_db(tickers: list[str], start_date: dt.date, end_date: dt.date) -> dict[str, dict[str, float]]:
    """Load short_sale_daily from DB. Returns {ticker: {date_str: short_ratio}}."""
    dsn = str(get_settings().postgres_dsn).replace("+asyncpg", "")
    conn = await asyncpg.connect(dsn)
    try:
        rows = await conn.fetch(
            """
            SELECT ticker_raw, trade_date::text AS date,
                   short_volume::float / NULLIF(total_volume::float, 0) AS ratio
            FROM short_sale_daily
            WHERE ticker_raw = ANY($1)
              AND trade_date >= $2
              AND trade_date <= $3
              AND total_volume IS NOT NULL
              AND total_volume > 0
            ORDER BY ticker_raw, trade_date
            """,
            tickers, start_date, end_date,
        )
    finally:
        await conn.close()

    result: dict[str, dict[str, float]] = {}
    for row in rows:
        ticker = row["ticker_raw"]
        if ticker not in result:
            result[ticker] = {}
        if row["ratio"] is not None:
            result[ticker][row["date"]] = float(row["ratio"])
    return result


def build_short_vol_db(tickers: list[str], trading_days: list[str]) -> dict[str, dict[str, float]]:
    start = dt.date.fromisoformat(trading_days[0]) - dt.timedelta(days=35)
    end = dt.date.fromisoformat(trading_days[-1])
    return asyncio.run(_load_short_vol_from_db(tickers, start, end))


# ── Short-Ratio Feature Computations ─────────────────────────────────────────

def compute_short_ratio_prior_day(ticker, entry_date, short_vol, trading_days):
    ticker_data = short_vol.get(ticker, {})
    if not ticker_data:
        return None
    if entry_date in trading_days:
        idx = trading_days.index(entry_date)
        for d in reversed(trading_days[:idx]):
            r = ticker_data.get(d)
            if r is not None:
                return r
    return None


def compute_short_ratio_avg_20d(ticker, entry_date, short_vol, trading_days):
    ticker_data = short_vol.get(ticker, {})
    if not ticker_data:
        return None
    if entry_date not in trading_days:
        return None
    idx = trading_days.index(entry_date)
    prior = trading_days[max(0, idx - 20):idx]
    vals = [ticker_data[d] for d in prior if d in ticker_data]
    if not vals:
        return None
    return sum(vals) / len(vals)


def compute_short_ratio_zscore_20d(ticker, entry_date, short_vol, trading_days):
    prior_day = compute_short_ratio_prior_day(ticker, entry_date, short_vol, trading_days)
    avg_20d = compute_short_ratio_avg_20d(ticker, entry_date, short_vol, trading_days)
    if prior_day is None or avg_20d is None:
        return None
    if entry_date not in trading_days:
        return None
    idx = trading_days.index(entry_date)
    prior = trading_days[max(0, idx - 20):idx]
    ticker_data = short_vol.get(ticker, {})
    vals = [ticker_data[d] for d in prior if d in ticker_data]
    if len(vals) < 5:
        return None
    std = statistics.stdev(vals) if len(vals) >= 2 else 0.0
    if std <= 0:
        return 0.0
    return (prior_day - avg_20d) / std


# ── Daily Bar Builder ─────────────────────────────────────────────────────────

import pyarrow.parquet as pq


def _build_daily_bar_from_intraday(path: Path, date: str) -> dict | None:
    try:
        table = pq.read_table(str(path))
        rows = table.to_pydict()
    except Exception:
        return None
    opens, highs, lows, closes, vols = [], [], [], [], []
    for i, ts_raw in enumerate(rows.get("timestamp", [])):
        try:
            if ts_raw.endswith("Z"):
                ts_raw = ts_raw[:-1] + "+00:00"
            ts = dt.datetime.fromisoformat(ts_raw).astimezone(_ET)
        except Exception:
            continue
        if _MKT_OPEN <= ts.time() < _MKT_CLOSE:
            opens.append(float(rows["open"][i] or 0))
            highs.append(float(rows["high"][i] or 0))
            lows.append(float(rows["low"][i] or 0))
            closes.append(float(rows["close"][i] or 0))
            vols.append(float(rows["volume"][i] or 0))
    if not opens:
        return None
    return {
        "date": date, "open": opens[0], "high": max(highs),
        "low": min(lows), "close": closes[-1], "volume": sum(vols),
    }


def build_daily_bars(tickers, dates, workers=8):
    root = Path(INTRADAY_CACHE_DIR)

    def _load(ticker):
        d_path = root / ticker
        if not d_path.is_dir():
            return ticker, []
        bars = []
        for date in dates:
            p = d_path / f"{date}.parquet"
            if not p.exists():
                continue
            bar = _build_daily_bar_from_intraday(p, date)
            if bar and bar["close"] > 0:
                bars.append(bar)
        return ticker, bars

    result = {}
    with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as ex:
        for ticker, bars in ex.map(_load, tickers):
            if bars:
                result[ticker] = bars
    return result


def load_intraday_bulk(candidates):
    import pandas as pd
    result = {}
    for date, tickers in candidates.items():
        day_bars = {}
        for ticker in tickers:
            p = Path(INTRADAY_CACHE_DIR) / ticker / f"{date}.parquet"
            if not p.exists():
                continue
            try:
                df = pd.read_parquet(str(p))
                if not df.empty and len(df) >= 5:
                    day_bars[ticker] = df.to_dict("records")
            except Exception:
                pass
        if day_bars:
            result[date] = day_bars
    return result


# ── Stats Helpers ─────────────────────────────────────────────────────────────

def pearson(xs, ys):
    if len(xs) != len(ys) or len(xs) < 2:
        return None
    n = len(xs)
    xm = sum(xs) / n
    ym = sum(ys) / n
    num = sum((xs[i] - xm) * (ys[i] - ym) for i in range(n))
    dx = sum((x - xm) ** 2 for x in xs) ** 0.5
    dy = sum((y - ym) ** 2 for y in ys) ** 0.5
    if dx <= 0 or dy <= 0:
        return None
    return num / (dx * dy)


def tercile_stats(vals, outcomes_r):
    if len(vals) < 6:
        return {}
    pairs = sorted(zip(vals, outcomes_r), key=lambda p: p[0])
    n = len(pairs)
    t = n // 3

    def stats(pairs_sub):
        ys = [p[1] for p in pairs_sub]
        wins = [y for y in ys if y > 0]
        wr = len(wins) / len(ys) if ys else 0.0
        avg = sum(ys) / len(ys) if ys else 0.0
        return {"n": len(ys), "wr": wr, "avg_r": avg}

    return {
        "low": stats(pairs[:t]),
        "mid": stats(pairs[t:2 * t]),
        "high": stats(pairs[2 * t:]),
    }


# ── Main ─────────────────────────────────────────────────────────────────────

def main():
    print("=== V25 Short-Volume Feature Diagnostic (V46 400d trade set) ===")
    print("STOP CONDITION: if no feature clears G2≥0.30R, V46 is terminal.\n")

    # 1. Load V46 config (but disable PEAD wiring in sim — no event prefetch here)
    with open(V46_CONFIG) as f:
        raw = yaml.safe_load(f)
    orb_params = dict(raw["orb_strategy"])
    # Disable PEAD in simulation (no event data available in diagnostic context)
    orb_params["prior_event_lookback_days"] = 0
    orb_params["weight_event_catalyst"] = 0.0
    params = ORBStrategyParams(**orb_params)
    print(f"Config: {V46_CONFIG} (PEAD disabled for diagnostic, V24-equivalent scoring)")
    print(f"weight_obv_slope={params.weight_obv_slope}")

    # 2. 400d trading window
    today = dt.date(2026, 4, 22)
    all_td = trading_days_between(today - dt.timedelta(days=700), today)
    trading_days_list = [d.isoformat() for d in all_td[-LOOKBACK_DAYS:]]
    print(f"Window: {trading_days_list[0]} → {trading_days_list[-1]} ({len(trading_days_list)} trading days)")

    extended_td = [d.isoformat() for d in all_td[-(LOOKBACK_DAYS + FEATURE_LOOKBACK_TRADING + 10):]]
    first_cal = dt.date.fromisoformat(extended_td[0])
    last_cal = dt.date.fromisoformat(trading_days_list[-1])
    needed_dates = []
    d = first_cal
    while d <= last_cal:
        needed_dates.append(d.isoformat())
        d += dt.timedelta(days=1)

    # 3. Load universe
    with open(UNIVERSE_FILE) as f:
        udata = yaml.safe_load(f)
    universe = udata.get("symbols", udata) if isinstance(udata, dict) else udata
    if "QQQ" not in universe:
        universe = list(universe) + ["QQQ"]
    print(f"Universe: {len(universe)} tickers")

    # 4. Build daily bars
    print(f"\nBuilding daily bars ({len(needed_dates)} calendar days)...")
    daily_bars = build_daily_bars(universe, needed_dates)
    print(f"Built daily bars for {len(daily_bars)} tickers")

    # 5. Enrichment
    print("Computing enrichment...")
    enrichment = enrich_daily_bars(daily_bars, trading_days_list)
    print(f"Enrichment for {len(enrichment)} tickers")

    # 6. Pre-screen + load intraday
    candidates = orb_pre_screen_candidates(
        daily_bars, trading_days_list, enrichment,
        min_price=params.min_price,
        min_atr=params.min_atr_14,
        min_avg_dollar_vol=params.min_avg_dollar_volume,
        max_per_day=None,
    )
    print("Loading intraday bars...")
    all_intraday = load_intraday_bulk(candidates)
    print(f"Loaded {sum(len(v) for v in all_intraday.values())} ticker-days")

    # 7. Run simulation (V24-equivalent)
    print("\nRunning simulation (V46 base params, PEAD disabled)...")
    state = ORBSimulationState(equity=params.initial_capital)
    day_results, _ = run_orb_simulation_with_state(
        all_intraday, trading_days_list, params, enrichment, state=state,
    )
    all_trades = [t for dr in day_results for t in dr.trades]
    trades_with_r = [t for t in all_trades if getattr(t, "r_multiple_at_exit", None) is not None]
    print(f"Trades: {len(all_trades)} total, {len(trades_with_r)} with r_multiple")

    if len(trades_with_r) < 40:
        print("ABORT: fewer than 40 trades — insufficient sample")
        return

    # 8. Load short volume from DB
    trade_tickers = list({t.ticker for t in trades_with_r})
    print(f"\nLoading short volume from DB for {len(trade_tickers)} tickers...")
    short_vol = build_short_vol_db(universe, trading_days_list)
    covered = sum(1 for t in trade_tickers if t in short_vol and short_vol[t])
    print(f"DB short-vol: {covered}/{len(trade_tickers)} trade tickers covered")

    # 9. Build OBV lookup
    sorted_daily = {t: sorted(bars, key=lambda b: b["date"]) for t, bars in daily_bars.items()}

    # 10. Compute features per trade
    print("\nComputing features per trade...")
    annotated = []
    missing = {"prior_day": 0, "avg_20d": 0, "zscore_20d": 0}

    for trade in trades_with_r:
        ticker = trade.ticker
        date = trade.date
        r = float(trade.r_multiple_at_exit)

        f_prior = compute_short_ratio_prior_day(ticker, date, short_vol, trading_days_list)
        f_avg = compute_short_ratio_avg_20d(ticker, date, short_vol, trading_days_list)
        f_zscore = compute_short_ratio_zscore_20d(ticker, date, short_vol, trading_days_list)

        bars_t = sorted_daily.get(ticker, [])
        prev_bars = [b for b in bars_t if b["date"][:10] < date]
        obv = compute_obv_slope_approx(prev_bars, lookback=20)

        if f_prior is None:
            missing["prior_day"] += 1
        if f_avg is None:
            missing["avg_20d"] += 1
        if f_zscore is None:
            missing["zscore_20d"] += 1

        annotated.append({
            "ticker": ticker, "date": date, "r": r,
            "prior_day": f_prior, "avg_20d": f_avg, "zscore_20d": f_zscore,
            "obv_slope": obv, "win": r > 0,
        })

    total = len(annotated)
    print(f"Annotated: {total} trades. Missing: {missing}")

    # 11. Feature analysis
    feature_defs = [
        ("short_ratio_prior_day", "prior_day", "low"),
        ("short_ratio_avg_20d", "avg_20d", "low"),
        ("short_ratio_zscore_20d", "zscore_20d", "low"),
    ]

    print("\n" + "=" * 90)
    print("FEATURE ANALYSIS — V46 400d trade set")
    print("=" * 90)

    results = {}
    for feat_name, feat_key, best_tercile in feature_defs:
        valid = [(t[feat_key], t["r"]) for t in annotated if t[feat_key] is not None]
        if len(valid) < 20:
            print(f"\n{feat_name}: SKIP — only {len(valid)} valid")
            results[feat_name] = None
            continue

        vals = [v[0] for v in valid]
        rs = [v[1] for v in valid]
        wins = [v for v in valid if v[1] > 0]

        rho = pearson(vals, rs)
        tstat = tercile_stats(vals, rs)
        coverage_pct = len(valid) / total

        print(f"\n{'─'*60}")
        print(f"FEATURE: {feat_name}")
        print(f"  n={len(valid)}/{total} ({coverage_pct*100:.0f}% coverage), overall WR={len(wins)/len(valid)*100:.1f}%")
        print(f"  Pearson = {rho:.4f}" if rho is not None else "  Pearson = n/a")

        worst_tercile = "high" if best_tercile == "low" else "low"
        if tstat:
            h = tstat["high"]
            m = tstat["mid"]
            lo = tstat["low"]
            print(f"  Tercile (low→high feature):")
            print(f"    Bottom: n={lo['n']}, WR={lo['wr']*100:.1f}%, avg_R={lo['avg_r']:+.3f}")
            print(f"    Middle: n={m['n']}, WR={m['wr']*100:.1f}%, avg_R={m['avg_r']:+.3f}")
            print(f"    Top:    n={h['n']}, WR={h['wr']*100:.1f}%, avg_R={h['avg_r']:+.3f}")

            best_st = tstat[best_tercile]
            worst_st = tstat[worst_tercile]
            rho_abs = abs(rho) if rho is not None else 0.0

            g1 = rho_abs >= 0.07 and len(valid) >= 120
            g2 = best_st["avg_r"] - worst_st["avg_r"] >= 0.30
            g3 = best_st["wr"] >= worst_st["wr"] + 0.05
            g4 = coverage_pct >= 0.50

            print(f"  G1 |Pearson|≥0.07 n≥120: {rho_abs:.4f}, n={len(valid)} → {'PASS ✓' if g1 else 'FAIL ✗'}")
            print(f"  G2 avg_R gap ≥ 0.30R: {best_st['avg_r']-worst_st['avg_r']:+.3f} → {'PASS ✓' if g2 else 'FAIL ✗'}")
            print(f"  G3 WR gap ≥ 5pp: {(best_st['wr']-worst_st['wr'])*100:+.1f}pp → {'PASS ✓' if g3 else 'FAIL ✗'}")
            print(f"  G4 coverage ≥ 50%: {coverage_pct*100:.0f}% → {'PASS ✓' if g4 else 'FAIL ✗'}")
            overall = g1 and g2 and g3 and g4
            print(f"  VERDICT: {'ALL GATES PASS → PROCEED TO PHASE 2' if overall else 'FAIL'}")
            results[feat_name] = {"pass": overall, "pearson": rho, "n": len(valid), "g2_delta": best_st["avg_r"]-worst_st["avg_r"]}

    # 12. G5 pairwise correlations
    print(f"\n{'─'*60}")
    print("G5 PAIRWISE CORRELATIONS (vs obv_slope_20):")
    for feat_key, short in [("prior_day","prior_day"),("avg_20d","avg_20d"),("zscore_20d","zscore_20d")]:
        combined = [(t[feat_key], t["obv_slope"]) for t in annotated
                    if t[feat_key] is not None and t["obv_slope"] is not None]
        if len(combined) >= 10:
            rho_x = pearson([c[0] for c in combined], [c[1] for c in combined])
            g5 = abs(rho_x) < 0.70 if rho_x is not None else True
            print(f"  ρ({short}, obv_slope) = {rho_x:.4f} → G5 {'PASS ✓' if g5 else 'FAIL ✗'}" if rho_x else f"  ρ({short}, obv_slope) = n/a")

    # 13. Summary
    passing = [n for n, r in results.items() if r and r["pass"]]
    print(f"\n{'='*90}")
    print("SUMMARY")
    print(f"{'='*90}")

    if passing:
        best = max(passing, key=lambda n: abs(results[n]["pearson"] or 0))
        print(f"PASS: {passing}. Best: {best} (Pearson={results[best]['pearson']:.4f})")
        print(f"→ PROCEED TO V25 WIRING (weight sign: NEGATIVE for 'low' best)")
    else:
        print("NO FEATURE PASSES ALL GATES.")
        # Check G2 specifically
        g2_vals = {n: r["g2_delta"] for n, r in results.items() if r}
        best_g2 = max(g2_vals, key=g2_vals.get) if g2_vals else None
        if best_g2:
            print(f"Best G2: {best_g2} = {g2_vals[best_g2]:+.3f}R (need ≥+0.30R)")
        print("→ V46 IS TERMINAL for this session. New data required for V47.")


if __name__ == "__main__":
    main()
