"""
Soft-Day Quality Gate Diagnostic (V23 ORB All-Weather Research)

Hypothesis: V20 tested stop/size tuning on soft-regime days and found WR=42.2%
(structural negative edge). BUT V20 did not test post-entry quality gates:
  (a) confirmation return >= +0.5% at 5 min after breakout entry
  (b) entropy cap: entropy_20d <= 0.86

If soft-day failures are caused by low-conviction / noisy breakouts that these
gates filter, WR could improve above 52%.

Method:
  1. Load V23 params; override market_regime_spy_threshold -> null (trade all days)
  2. Build synthetic daily bars from intraday cache (avoids sparse daily parquet coverage)
  3. Compute enrichment (entropy_20d, QQQ gap, ATR, etc.)
  4. Pre-screen candidates; load intraday bars; run simulation
  5. Tag each trade: HARD (QQQ >= +0.15%), SOFT (0.0-0.15%), NEG (< 0.0)
  6. Compute confirm_return (5 min after entry), entropy (from enrichment)
  7. Print 4-arm x 3-bucket matrix + gate verdicts

Gates (SOFT bucket, Arm D):
  G1: WR >= 52%          (V20 baseline was 42.2%)
  G2: avg_win/loss >= 1.10
  G3: trades >= 30
  G4 (cardinal): HARD Arm D WR >= HARD Arm A WR - 5pp (gates don't hurt V23 native days)
"""
from __future__ import annotations

import datetime as dt
import json
import os
import sys
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from zoneinfo import ZoneInfo

import numpy as np
import pandas as pd
import pyarrow.parquet as pq
import yaml

# Add project root to path
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../..")))

from libs.intraday.domain import ORBStrategyParams
from libs.intraday.features import enrich_daily_bars
from libs.intraday.orb_simulator import ORBSimulationState, run_orb_simulation_with_state
from libs.intraday.screener import orb_pre_screen_candidates

# ── Config ─────────────────────────────────────────────────────────────────
V23_CONFIG = "configs/intraday/strategies/orb_gainers_v23.yaml"
V23_BASELINE_RUN = "runs/intraday_orb/intraday_20260420_205136_5597b16d.json"
UNIVERSE_FILE = "configs/symbols_midlarge_snapshot_exact.yaml"
INTRADAY_CACHE_DIR = "data/cache/intraday"

HARD_THRESHOLD = 0.0015   # QQQ gap >= +0.15% = HARD
SOFT_LOWER = 0.0          # QQQ gap 0-0.15% = SOFT; < 0% = NEG

CONFIRM_THRESHOLD = 0.005  # >= +0.5% confirmation
ENTROPY_CAP = 0.86

_ET = ZoneInfo("America/New_York")
_MKT_OPEN = dt.time(9, 30)
_MKT_CLOSE = dt.time(16, 0)


# ── Synthetic Daily Bar Builder ─────────────────────────────────────────────

def _build_daily_bar_from_intraday_file(path: Path, date: str) -> dict | None:
    """Build one daily OHLCV bar from a market-hours intraday parquet file."""
    try:
        table = pq.read_table(str(path))
        rows = table.to_pydict()
    except Exception:
        return None

    mkt_opens, mkt_highs, mkt_lows, mkt_closes, mkt_vols = [], [], [], [], []
    timestamps = rows.get("timestamp", [])
    for i in range(len(timestamps)):
        ts_raw = timestamps[i]
        try:
            if ts_raw.endswith("Z"):
                ts_raw = ts_raw[:-1] + "+00:00"
            ts = dt.datetime.fromisoformat(ts_raw).astimezone(_ET)
        except Exception:
            continue
        if _MKT_OPEN <= ts.time() < _MKT_CLOSE:
            mkt_opens.append(float(rows["open"][i] or 0))
            mkt_highs.append(float(rows["high"][i] or 0))
            mkt_lows.append(float(rows["low"][i] or 0))
            mkt_closes.append(float(rows["close"][i] or 0))
            mkt_vols.append(float(rows["volume"][i] or 0))

    if not mkt_opens:
        return None
    return {
        "date": date,
        "open": mkt_opens[0],
        "high": max(mkt_highs),
        "low": min(mkt_lows),
        "close": mkt_closes[-1],
        "volume": sum(mkt_vols),
    }


def build_daily_bars_from_intraday(
    tickers: list[str],
    needed_dates: list[str],
    cache_dir: str = INTRADAY_CACHE_DIR,
    workers: int = 8,
) -> dict[str, list[dict]]:
    """Build synthetic daily OHLCV bars from intraday cache for the needed date range.

    Uses parallel loading for speed. Returns {ticker: [daily_bar_dict, ...]}.
    """
    root = Path(cache_dir)
    needed_set = set(needed_dates)

    def _load_ticker(ticker: str) -> tuple[str, list[dict]]:
        ticker_dir = root / ticker
        if not ticker_dir.is_dir():
            return ticker, []
        bars: list[dict] = []
        for date in needed_dates:
            path = ticker_dir / f"{date}.parquet"
            if not path.exists():
                continue
            bar = _build_daily_bar_from_intraday_file(path, date)
            if bar and bar["close"] > 0:
                bars.append(bar)
        return ticker, bars

    result: dict[str, list[dict]] = {}
    with ThreadPoolExecutor(max_workers=workers) as executor:
        futures = {executor.submit(_load_ticker, t): t for t in tickers}
        done = 0
        for future in futures:
            ticker, bars = future.result()
            if bars:
                result[ticker] = bars
            done += 1
            if done % 100 == 0:
                print(f"  Built daily bars: {done}/{len(tickers)} tickers", flush=True)
    return result


# ── Intraday Bar Loader ─────────────────────────────────────────────────────

def load_intraday_bars(ticker: str, date: str) -> list[dict] | None:
    path = Path(INTRADAY_CACHE_DIR) / ticker / f"{date}.parquet"
    if not path.exists():
        return None
    try:
        df = pd.read_parquet(str(path))
        if df.empty or len(df) < 5:
            return None
        return df.to_dict("records")
    except Exception:
        return None


def load_intraday_bulk(candidates: dict[str, list[str]]) -> dict[str, dict[str, list[dict]]]:
    result: dict[str, dict[str, list[dict]]] = {}
    pairs = sum(len(v) for v in candidates.values())
    loaded = 0
    for date, tickers in candidates.items():
        day_bars: dict[str, list[dict]] = {}
        for ticker in tickers:
            bars = load_intraday_bars(ticker, date)
            if bars is not None:
                day_bars[ticker] = bars
            loaded += 1
        if day_bars:
            result[date] = day_bars
        if loaded % 2000 == 0 and loaded > 0:
            print(f"  Loaded {loaded}/{pairs} intraday files...", flush=True)
    return result


# ── Universe Loader ─────────────────────────────────────────────────────────

def load_universe() -> list[str]:
    with open(UNIVERSE_FILE) as f:
        data = yaml.safe_load(f)
    symbols = data.get("symbols", data) if isinstance(data, dict) else data
    if "QQQ" not in symbols:
        symbols = list(symbols) + ["QQQ"]
    return symbols


# ── Confirmation Return ─────────────────────────────────────────────────────

def compute_confirm_return(
    ticker: str,
    date: str,
    entry_time_utc: str,
    entry_price: float,
    confirm_minutes: int = 5,
) -> float | None:
    bars = load_intraday_bars(ticker, date)
    if not bars:
        return None
    try:
        ts_raw = entry_time_utc
        if ts_raw.endswith("Z"):
            ts_raw = ts_raw[:-1] + "+00:00"
        entry_et = dt.datetime.fromisoformat(ts_raw).astimezone(_ET)
    except Exception:
        return None

    target_ts = entry_et + dt.timedelta(minutes=confirm_minutes)
    for bar in bars:
        try:
            b_raw = bar["timestamp"]
            if b_raw.endswith("Z"):
                b_raw = b_raw[:-1] + "+00:00"
            bar_ts = dt.datetime.fromisoformat(b_raw).astimezone(_ET)
        except Exception:
            continue
        if bar_ts >= target_ts:
            close = float(bar.get("close") or 0)
            if close > 0 and entry_price > 0:
                return (close - entry_price) / entry_price
            return None
    return None


# ── Analysis ────────────────────────────────────────────────────────────────

def classify_regime(qqq_gap: float | None) -> str:
    if qqq_gap is None:
        return "unknown"
    if qqq_gap >= HARD_THRESHOLD:
        return "HARD"
    if qqq_gap >= SOFT_LOWER:
        return "SOFT"
    return "NEG"


def arm_stats(trades: list[dict]) -> dict:
    if not trades:
        return {"n": 0, "wr": 0.0, "avg_win": 0.0, "avg_loss": 0.0, "wl": 0.0, "pnl": 0.0}
    wins = [t for t in trades if t["pnl"] > 0]
    losses = [t for t in trades if t["pnl"] <= 0]
    wr = len(wins) / len(trades)
    avg_win = float(np.mean([t["pnl"] for t in wins])) if wins else 0.0
    avg_loss = float(np.mean([abs(t["pnl"]) for t in losses])) if losses else 0.0
    wl = avg_win / avg_loss if avg_loss > 0 else 0.0
    pnl = sum(t["pnl"] for t in trades)
    return {"n": len(trades), "wr": wr, "avg_win": avg_win, "avg_loss": avg_loss, "wl": wl, "pnl": pnl}


# ── Main ────────────────────────────────────────────────────────────────────

def main() -> None:
    print("=== V23 Soft-Day Quality Gate Diagnostic ===\n")

    # 1. Load V23 config + override regime
    with open(V23_CONFIG) as f:
        raw = yaml.safe_load(f)
    orb_params = ORBStrategyParams(**raw["orb_strategy"])
    orb_params_noregime = orb_params.model_copy(update={
        "market_regime_spy_threshold": None,
        "regime_skip_below": None,
        "regime_size_scale_low": None,
        "regime_size_scale_high": None,
        "regime_size_scale_min": None,
    })
    print(f"V23 regime threshold: {orb_params.market_regime_spy_threshold} → overridden to null")

    # 2. Load V23 baseline for window
    with open(V23_BASELINE_RUN) as f:
        v23_data = json.load(f)
    trading_days = [d["date"] for d in v23_data["daily_summary"]]
    print(f"Window: {trading_days[0]} → {trading_days[-1]} ({len(trading_days)} days)")

    # 3. Build needed dates for enrichment (30 prior days for avg_dollar_vol_30d)
    all_dates_sorted = sorted(set(trading_days))
    first_day = dt.date.fromisoformat(all_dates_sorted[0])
    last_day = dt.date.fromisoformat(all_dates_sorted[-1])

    # Generate all dates in lookback window (including weekends for filtering)
    lookback_start = first_day - dt.timedelta(days=60)  # ~30 trading days
    needed_dates: list[str] = []
    d = lookback_start
    while d <= last_day:
        needed_dates.append(d.isoformat())
        d += dt.timedelta(days=1)
    needed_dates_set = set(needed_dates)
    print(f"Building daily bars for {len(needed_dates)} calendar dates ({lookback_start} → {last_day})")

    # 4. Build synthetic daily bars from intraday cache
    universe = load_universe()
    print(f"\nBuilding synthetic daily bars from intraday cache for {len(universe)} tickers...")
    daily_bars = build_daily_bars_from_intraday(universe, needed_dates)
    print(f"Built daily bars for {len(daily_bars)} tickers")

    # 5. Compute enrichment
    print("Computing enrichment...")
    enrichment = enrich_daily_bars(daily_bars, trading_days)
    print(f"Enrichment computed for {len(enrichment)} tickers")

    # QQQ gap classification
    qqq_gaps: dict[str, float | None] = {}
    for day in trading_days:
        qe = enrichment.get("QQQ", {}).get(day, {})
        pc = qe.get("prev_close")
        to = qe.get("today_open")
        qqq_gaps[day] = (to - pc) / pc if (pc and to and pc > 0) else None

    hard_days = sum(1 for d in trading_days if classify_regime(qqq_gaps.get(d)) == "HARD")
    soft_days = sum(1 for d in trading_days if classify_regime(qqq_gaps.get(d)) == "SOFT")
    neg_days  = sum(1 for d in trading_days if classify_regime(qqq_gaps.get(d)) == "NEG")
    print(f"Day classification: HARD={hard_days}, SOFT={soft_days}, NEG={neg_days}")

    # 6. Pre-screen candidates
    print("\nPre-screening candidates...")
    candidates = orb_pre_screen_candidates(
        daily_bars,
        trading_days,
        enrichment,
        min_price=orb_params.min_price,
        min_atr=orb_params.min_atr_14,
        min_avg_dollar_vol=orb_params.min_avg_dollar_volume,
        max_per_day=None,
    )
    total_pairs = sum(len(v) for v in candidates.values())
    avg_per_day = total_pairs / len(trading_days) if trading_days else 0
    print(f"Pre-screened: {total_pairs} ticker-days ({avg_per_day:.0f} avg/day)")

    # 7. Load intraday bars for pre-screened candidates
    print(f"\nLoading intraday bars ({total_pairs} files)...")
    all_intraday = load_intraday_bulk(candidates)
    intraday_pairs = sum(len(v) for v in all_intraday.values())
    hit_rate = intraday_pairs / total_pairs * 100 if total_pairs > 0 else 0
    print(f"Loaded: {intraday_pairs} ticker-days (hit rate: {hit_rate:.0f}%)")

    # 8. Run simulation with NO regime filter
    print("\nRunning V23 simulation (no regime filter)...")
    state = ORBSimulationState(equity=orb_params.initial_capital)
    day_results, _ = run_orb_simulation_with_state(
        all_intraday,
        trading_days,
        orb_params_noregime,
        enrichment,
        state=state,
    )
    all_trades_raw = [t for dr in day_results for t in dr.trades]
    trade_days = sum(1 for dr in day_results if dr.trades)
    print(f"Total trades: {len(all_trades_raw)} across {trade_days} days")
    skip_counts: dict[str, int] = {}
    for dr in day_results:
        if dr.skip_reason:
            skip_counts[dr.skip_reason] = skip_counts.get(dr.skip_reason, 0) + 1
    print(f"Skip breakdown: {skip_counts}")

    # Sanity check: compare HARD days to V23 baseline
    print(f"V23 baseline: {len(v23_data['trades'])} trades (reference)")

    # 9. Annotate each trade: regime bucket, entropy, confirm_return
    print("\nAnnotating trades...")
    annotated: list[dict] = []
    for trade in all_trades_raw:
        ticker = trade.ticker
        date = trade.date
        qqq_gap = qqq_gaps.get(date)
        bucket = classify_regime(qqq_gap)
        entropy = enrichment.get(ticker, {}).get(date, {}).get("entropy_20d")
        entry_time = getattr(trade, "entry_time", None) or ""
        entry_price = getattr(trade, "entry_price", 0.0) or 0.0
        confirm_ret = compute_confirm_return(ticker, date, entry_time, entry_price, 5)
        annotated.append({
            "ticker": ticker, "date": date, "pnl": trade.pnl,
            "bucket": bucket, "qqq_gap": qqq_gap,
            "entropy": entropy, "confirm_ret": confirm_ret,
            "win": trade.pnl > 0, "exit_reason": trade.exit_reason,
        })
    print(f"Annotated {len(annotated)} trades")
    missing_confirm = sum(1 for t in annotated if t["confirm_ret"] is None)
    missing_entropy = sum(1 for t in annotated if t["entropy"] is None)
    print(f"  Missing confirm_ret: {missing_confirm}")
    print(f"  Missing entropy:     {missing_entropy}")

    # 10. 4-arm x 3-bucket matrix
    print("\n" + "=" * 105)
    print("4-ARM × 3-BUCKET MATRIX")
    print("  Arm A: all trades  |  Arm B: confirm≥0.5%  |  Arm C: entropy≤0.86  |  Arm D: B∩C")
    print("=" * 105)
    col_w = 30

    hard_arm_a_wr = 0.0
    hard_arm_d_wr = 0.0
    soft_arm_d: dict = {}

    for bucket in ["HARD", "SOFT", "NEG"]:
        pool = [t for t in annotated if t["bucket"] == bucket]
        arm_a = pool
        arm_b = [t for t in pool if t["confirm_ret"] is not None and t["confirm_ret"] >= CONFIRM_THRESHOLD]
        arm_c = [t for t in pool if t["entropy"] is not None and t["entropy"] <= ENTROPY_CAP]
        arm_d = [t for t in pool if
                 t["confirm_ret"] is not None and t["confirm_ret"] >= CONFIRM_THRESHOLD
                 and t["entropy"] is not None and t["entropy"] <= ENTROPY_CAP]
        sa, sb, sc, sd = arm_stats(arm_a), arm_stats(arm_b), arm_stats(arm_c), arm_stats(arm_d)

        def fmt(s: dict) -> str:
            if s["n"] == 0:
                return f"  n=  0  WR= 0%  W/L=n/a  ${0:+7.0f}  "
            return f"  n={s['n']:3d}  WR={s['wr']*100:.0f}%  W/L={s['wl']:.2f}  ${s['pnl']:+7.0f}  "

        print(f"{bucket:4s}  |{fmt(sa)}|{fmt(sb)}|{fmt(sc)}|{fmt(sd)}")

        if bucket == "HARD":
            hard_arm_a_wr = sa["wr"]
            hard_arm_d_wr = sd["wr"] if sd["n"] > 0 else 0.0
        elif bucket == "SOFT":
            soft_arm_d = sd

    print("=" * 105)

    # 11. Gate evaluation
    print("\n=== GATE EVALUATION (SOFT bucket, Arm D) ===")
    if soft_arm_d.get("n", 0) == 0:
        print("Arm D: 0 trades — G3 FAIL (no tradeable soft-day sample with both gates)")
        print("\nVERDICT: ABORT — quality gates produce no soft-day sample; V23 terminal re-confirmed")
        _print_context(annotated)
        return

    sd = soft_arm_d
    g1 = sd["wr"] >= 0.52
    g2 = (sd["wl"] >= 1.10) if sd["avg_loss"] > 0 else False
    g3 = sd["n"] >= 30
    g4 = hard_arm_d_wr >= hard_arm_a_wr - 0.05  # gates don't hurt HARD days

    print(f"G1 (SOFT Arm D WR ≥ 52%):              WR={sd['wr']*100:.1f}%   → {'PASS' if g1 else 'FAIL'}")
    print(f"G2 (SOFT Arm D W/L ≥ 1.10):            W/L={sd['wl']:.2f}    → {'PASS' if g2 else 'FAIL'}")
    print(f"G3 (SOFT Arm D trades ≥ 30):            n={sd['n']}       → {'PASS' if g3 else 'FAIL'}")
    print(f"G4 (HARD Arm D WR ≥ HARD Arm A WR-5pp): {hard_arm_d_wr*100:.1f}% vs {hard_arm_a_wr*100:.1f}%  → {'PASS' if g4 else 'FAIL'}")

    all_pass = g1 and g2 and g3 and g4
    if all_pass:
        print("\nVERDICT: PROCEED TO PHASE 2 — build orb_gainers_v24_allweather.yaml")
    else:
        failed = [f"G{i+1}" for i, g in enumerate([g1, g2, g3, g4]) if not g]
        print(f"\nVERDICT: ABORT — failed gates {', '.join(failed)}; V23 terminal re-confirmed")

    _print_context(annotated)


def _print_context(annotated: list[dict]) -> None:
    print("\n=== CONTEXT ===")
    for bucket in ["HARD", "SOFT", "NEG"]:
        pool = [t for t in annotated if t["bucket"] == bucket]
        if not pool:
            print(f"{bucket}: 0 trades")
            continue
        wr = sum(1 for t in pool if t["win"]) / len(pool)
        print(f"{bucket}: n={len(pool)}, WR={wr*100:.0f}%")
        # Show confirm_ret distribution
        conf_vals = [t["confirm_ret"] for t in pool if t["confirm_ret"] is not None]
        if conf_vals:
            pass_pct = sum(1 for v in conf_vals if v >= CONFIRM_THRESHOLD) / len(conf_vals) * 100
            print(f"  confirm_ret: {len(conf_vals)}/{len(pool)} have data, {pass_pct:.0f}% pass 0.5% threshold")
        entr_vals = [t["entropy"] for t in pool if t["entropy"] is not None]
        if entr_vals:
            pass_pct = sum(1 for v in entr_vals if v <= ENTROPY_CAP) / len(entr_vals) * 100
            print(f"  entropy_20d: {len(entr_vals)}/{len(pool)} have data, {pass_pct:.0f}% pass ≤0.86 cap")


if __name__ == "__main__":
    main()
