"""
V47 Diagnostic: Form 4 Insider Buy Signal

Hypothesis: Open-market insider purchases (SEC Form 4, code P) filed before ORB
entry predict follow-through. Based on Lakonishok/Lee (JF 2001) and
Jeng/Metrick/Zeckhauser (RFS 2003): CEO/CFO cluster buys predict +6-11%
12-month excess returns.

Features tested:
  insider_buy_count_30d      : P-code buy count filed in prior 30 calendar days
  insider_buy_dollar_30d     : log1p(total $ value) in prior 30 days
  insider_cluster_30d        : unique insider CIK count in prior 30 days (cluster size)
  insider_csuite_count_60d   : C-suite (CEO/CFO/etc) buys filed in prior 60 days
  insider_buy_pct_holding_30d: mean(purchase_pct_of_holding) in prior 30 days
  insider_recency            : calendar days since most recent buy (sentinel=90 if none)

Gates (pre-committed):
  G1: |Pearson| >= 0.07 AND n >= 200
  G2: |top - bottom tercile avg_R| >= 0.30R  <- binding gate
  G3: |top - bottom tercile WR| >= 5pp
  G4: coverage >= 30%  (sparse signal; lower than usual 80%)
  G5a: |rho(feature, obv_slope_20)| < 0.70
  G5b: |rho(feature, raw gap_pct)| < 0.70
  G5c: |rho(feature, PEAD_prior_d7)| < 0.70

Decision rule (pre-committed):
  >= 1 feature passes G1-G5c -> advance highest-G2 to Phase 2 backtest
  0 pass -> close axis, document in lineage memory

Data source: Oracle API http://localhost:18001 (Form 4, P-code only)
V46 400d baseline: runs/v46_correct_w12_400d.json/intraday_20260422_043215_63f03e64.json
"""
from __future__ import annotations

import json
import math
import os
import sys
from collections import defaultdict
from pathlib import Path

import datetime as dt
import requests

sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../..")))

V46_400D_RUN = "runs/v46_correct_w12_400d.json/intraday_20260422_043215_63f03e64.json"
ORACLE_BASE = "http://localhost:18001"

RECENCY_SENTINEL = 90   # days; used when no buy found in 90-day lookback
G4_THRESHOLD = 0.30     # sparse signal; lower than usual 80%


# ── Oracle helpers ──────────────────────────────────────────────────────────

def _fetch_form4(ticker: str, start: str) -> list[dict]:
    """Fetch all P-code Form 4 buys for a ticker since `start` (buy_only=P-code).
    PIT enforcement is done locally by filtering filing_date < entry_date.
    """
    url = f"{ORACLE_BASE}/api/v1/insider/form4/{ticker}"
    params = {
        "as_of": "2026-12-31",  # broad as_of; we filter locally per trade
        "start": start,
        "buy_only": "true",
    }
    try:
        r = requests.get(url, params=params, timeout=30)
        r.raise_for_status()
        return r.json().get("transactions", [])
    except Exception as e:
        print(f"  WARN: {ticker} fetch failed: {e}")
        return []


# ── Statistical helpers ─────────────────────────────────────────────────────

def pearson(xs: list[float], ys: list[float]) -> float | None:
    n = len(xs)
    if n < 5:
        return None
    xm, ym = sum(xs) / n, sum(ys) / n
    num = sum((xs[i] - xm) * (ys[i] - ym) for i in range(n))
    dx = sum((x - xm) ** 2 for x in xs) ** 0.5
    dy = sum((y - ym) ** 2 for y in ys) ** 0.5
    if dx <= 0 or dy <= 0:
        return None
    return num / (dx * dy)


def tercile_stats(vals: list[float], rs: list[float]) -> dict:
    if len(vals) < 9:
        return {}
    pairs = sorted(zip(vals, rs), key=lambda p: p[0])
    n = len(pairs)
    t = n // 3

    def stats(sub):
        ys = [p[1] for p in sub]
        return {"n": len(ys), "wr": sum(1 for y in ys if y > 0) / len(ys), "avg_r": sum(ys) / len(ys)}

    return {"low": stats(pairs[:t]), "mid": stats(pairs[t : 2 * t]), "high": stats(pairs[2 * t :])}


def pairwise_rho(
    idxs_a: list[int], vals_a: list[float],
    idxs_b: list[int], vals_b: list[float],
) -> float | None:
    map_b = {idx: vals_b[i] for i, idx in enumerate(idxs_b)}
    va, vb = [], []
    for i, idx in enumerate(idxs_a):
        if idx in map_b:
            va.append(vals_a[i])
            vb.append(map_b[idx])
    return pearson(va, vb) if len(va) >= 10 else None


def report_feature(
    label: str,
    idxs: list[int],
    vals: list[float],
    rs: list[float],
    n_total: int,
    g5_comparisons: list[tuple[str, list[int], list[float]]],
) -> dict:
    n = len(vals)
    p = pearson(vals, rs)
    ts = tercile_stats(vals, rs)
    coverage = n / n_total

    g4 = coverage >= G4_THRESHOLD

    if not ts or p is None:
        print(f"  {label}: n={n}/{n_total} ({coverage:.1%})  INSUFFICIENT DATA")
        return {"pass_g2": False, "pass_all": False, "pearson": None, "g2_gap": None}

    low, mid, high = ts["low"], ts["mid"], ts["high"]
    raw_g2 = high["avg_r"] - low["avg_r"]
    avg_r_gap = abs(raw_g2)
    wr_gap = abs(high["wr"] - low["wr"])

    g1 = abs(p) >= 0.07 and n >= 200
    g2 = avg_r_gap >= 0.30
    g3 = wr_gap >= 0.05

    print(f"\n  [{label}]  n={n}/{n_total} ({coverage:.1%})  Pearson={p:+.3f}  direction={'↑high' if raw_g2 > 0 else '↓low'}")
    print(f"    G1: {'PASS' if g1 else 'FAIL'} (|{abs(p):.3f}| {'≥' if abs(p) >= 0.07 else '<'} 0.07, n={n} {'≥' if n >= 200 else '<'} 200)")
    print(f"    G2: {'PASS' if g2 else 'FAIL'} (avg_R gap = {avg_r_gap:.3f}R  [threshold 0.30R])")
    print(f"    G3: {'PASS' if g3 else 'FAIL'} (WR gap = {wr_gap * 100:.1f}pp  [threshold 5pp])")
    print(f"    G4: {'PASS' if g4 else 'FAIL'} (coverage = {coverage:.1%}  [threshold {G4_THRESHOLD:.0%}])")
    print(f"    Bottom tercile: WR {low['wr'] * 100:.1f}%  avg_R {low['avg_r']:+.3f}  n={low['n']}")
    print(f"    Middle tercile: WR {mid['wr'] * 100:.1f}%  avg_R {mid['avg_r']:+.3f}  n={mid['n']}")
    print(f"    Top tercile:    WR {high['wr'] * 100:.1f}%  avg_R {high['avg_r']:+.3f}  n={high['n']}")

    g5_results: dict = {}
    pass_all_g5 = True
    for cmp_label, cmp_idxs, cmp_vals in g5_comparisons:
        rho = pairwise_rho(idxs, vals, cmp_idxs, cmp_vals)
        if rho is not None:
            gate_pass = abs(rho) < 0.70
            g5_results[f"g5_{cmp_label}"] = {"rho": rho, "pass": gate_pass}
            print(f"    G5({cmp_label}): {'PASS' if gate_pass else 'FAIL'} (ρ={rho:+.3f}  [|ρ|<0.70])")
            if not gate_pass:
                pass_all_g5 = False

    pass_all = g1 and g2 and g3 and g4 and pass_all_g5
    print(f"    → {'ALL GATES PASS ✓' if pass_all else 'FAIL'}")

    return {
        "pass_g2": g2, "pass_all": pass_all,
        "pearson": p, "g2_gap": avg_r_gap, "raw_g2_gap": raw_g2,
        "direction": "high" if raw_g2 > 0 else "low",
        **g5_results,
    }


# ── Feature computation ─────────────────────────────────────────────────────

def _compute_features(
    filing_date_str: str,
    entry_date_str: str,
    ticker_buys: list[dict],
) -> dict[str, float | None]:
    """Compute all 6 features for a single trade.

    PIT rule: only filings where filing_date < entry_date (strict).
    """
    entry_dt = dt.date.fromisoformat(entry_date_str)
    cutoff_30 = entry_dt - dt.timedelta(days=30)
    cutoff_60 = entry_dt - dt.timedelta(days=60)
    cutoff_90 = entry_dt - dt.timedelta(days=RECENCY_SENTINEL)

    # Filter: filing_date strictly < entry_date
    prior = [
        t for t in ticker_buys
        if dt.date.fromisoformat(t["filing_date"]) < entry_dt
    ]

    # 30-day window
    w30 = [t for t in prior if dt.date.fromisoformat(t["filing_date"]) >= cutoff_30]
    # 60-day window
    w60 = [t for t in prior if dt.date.fromisoformat(t["filing_date"]) >= cutoff_60]
    # 90-day window (for recency)
    w90 = [t for t in prior if dt.date.fromisoformat(t["filing_date"]) >= cutoff_90]

    # Feature 1: buy count in 30d
    buy_count_30d = len(w30) if w30 else None

    # Feature 2: log1p(dollar total) in 30d (only include buys with value > 0)
    dollar_vals = [t["total_value"] for t in w30 if t.get("total_value", 0) > 0]
    buy_dollar_30d = math.log1p(sum(dollar_vals)) if dollar_vals else None

    # Feature 3: cluster size (unique CIKs) in 30d
    ciks = set(t["owner_cik"] for t in w30 if t.get("owner_cik"))
    cluster_30d = len(ciks) if w30 else None

    # Feature 4: c-suite count in 60d
    csuite_60d = sum(1 for t in w60 if t.get("is_c_suite")) if w60 else None

    # Feature 5: avg purchase_pct_of_holding in 30d
    pcts = [t["purchase_pct_of_holding"] for t in w30 if t.get("purchase_pct_of_holding") is not None]
    buy_pct_holding_30d = sum(pcts) / len(pcts) if pcts else None

    # Feature 6: recency (days since most recent buy), sentinel if none in 90d
    if w90:
        most_recent = max(dt.date.fromisoformat(t["filing_date"]) for t in w90)
        recency = (entry_dt - most_recent).days
    else:
        recency = RECENCY_SENTINEL  # always defined

    return {
        "insider_buy_count_30d": buy_count_30d,
        "insider_buy_dollar_30d": buy_dollar_30d,
        "insider_cluster_30d": cluster_30d,
        "insider_csuite_count_60d": csuite_60d,
        "insider_buy_pct_holding_30d": buy_pct_holding_30d,
        "insider_recency": float(recency),
    }


# ── Main ───────────────────────────────────────────────────────────────────

def main() -> None:
    print("=== V47 Diagnostic: Form 4 Insider Buy Signal ===\n")
    print("Pre-committed gates: G1(|P|≥0.07,n≥200) G2(avg_R≥0.30R) G3(WR≥5pp)"
          f" G4(cov≥{G4_THRESHOLD:.0%}) G5a/b/c(|ρ|<0.70)")
    print("Pre-committed decision rule: highest G2 that passes ALL gates → advance to Phase 2\n")

    with open(V46_400D_RUN) as f:
        run_data = json.load(f)
    trades_raw = run_data.get("trades", [])
    m = run_data.get("metrics", {})
    print(f"V46 400d: {m.get('total_trades')} trades, {m.get('start_date')} → {m.get('end_date')}, "
          f"return={m.get('total_return_pct', 0) * 100:.1f}%\n")

    trades = [
        {
            "idx": i,
            "ticker": t["ticker"],
            "date": t["date"][:10],
            "r": float(t["r_multiple_at_exit"]),
            "obv_slope": t.get("obv_slope_20") or 0.0,
            "gap_pct": t.get("gap_pct") or 0.0,
            "pead_prior": t.get("weight_event_catalyst") or t.get("pead_prior_d7_score") or 0.0,
        }
        for i, t in enumerate(trades_raw)
        if t.get("r_multiple_at_exit") is not None
    ]
    n_total = len(trades)
    print(f"Valid trades (with r_multiple): {n_total}")

    # ── Fetch Form 4 data per unique ticker ──────────────────────────────
    unique_tickers = sorted(set(t["ticker"] for t in trades))
    earliest_date = min(t["date"] for t in trades)
    fetch_start = (dt.date.fromisoformat(earliest_date) - dt.timedelta(days=95)).isoformat()

    print(f"\nFetching Form 4 P-code buys from Oracle for {len(unique_tickers)} tickers (start={fetch_start})...")
    form4_by_ticker: dict[str, list[dict]] = {}
    for i, ticker in enumerate(unique_tickers):
        buys = _fetch_form4(ticker, fetch_start)
        form4_by_ticker[ticker] = buys
        if (i + 1) % 20 == 0:
            print(f"  {i + 1}/{len(unique_tickers)} fetched...")

    total_buys = sum(len(v) for v in form4_by_ticker.values())
    tickers_with_buys = sum(1 for v in form4_by_ticker.values() if v)
    print(f"  Done. {total_buys} P-code buy events across {tickers_with_buys}/{len(unique_tickers)} tickers\n")

    # ── Compute features per trade ────────────────────────────────────────
    print("Computing features per trade...")

    feat_arrays: dict[str, list[tuple[int, float]]] = {
        "insider_buy_count_30d": [],
        "insider_buy_dollar_30d": [],
        "insider_cluster_30d": [],
        "insider_csuite_count_60d": [],
        "insider_buy_pct_holding_30d": [],
        "insider_recency": [],
    }
    rs_all = [t["r"] for t in trades]
    idxs_all = [t["idx"] for t in trades]

    # G5 reference arrays (obv_slope, gap_pct, pead)
    obv_idxs = [t["idx"] for t in trades]
    obv_vals = [t["obv_slope"] for t in trades]
    gap_idxs = [t["idx"] for t in trades]
    gap_vals = [t["gap_pct"] for t in trades]
    pead_idxs = [t["idx"] for t in trades]
    pead_vals = [t["pead_prior"] for t in trades]

    for trade in trades:
        ticker = trade["ticker"]
        buys = form4_by_ticker.get(ticker, [])
        feats = _compute_features(trade["date"], trade["date"], buys)

        for feat_name, val in feats.items():
            if val is not None:
                feat_arrays[feat_name].append((trade["idx"], val))

    # ── Report ────────────────────────────────────────────────────────────
    print("\n" + "=" * 72)
    print("FEATURE DIAGNOSTICS")
    print("=" * 72)

    g5_comparisons = [
        ("obv_slope", obv_idxs, obv_vals),
        ("raw_gap", gap_idxs, gap_vals),
        ("pead_prior", pead_idxs, pead_vals),
    ]

    results: dict[str, dict] = {}
    for feat_name, pairs in feat_arrays.items():
        idxs = [p[0] for p in pairs]
        vals = [p[1] for p in pairs]
        rs = [rs_all[idxs_all.index(idx)] for idx in idxs]
        results[feat_name] = report_feature(feat_name, idxs, vals, rs, n_total, g5_comparisons)

    # ── Decision ─────────────────────────────────────────────────────────
    print("\n" + "=" * 72)
    print("DECISION")
    print("=" * 72)

    passers = [(name, r) for name, r in results.items() if r.get("pass_all")]
    closers = [(name, r) for name, r in results.items() if not r.get("pass_all")]

    if passers:
        best = max(passers, key=lambda x: x[1].get("g2_gap") or 0.0)
        print(f"\n✓ {len(passers)} feature(s) passed all gates.")
        print(f"  Best G2: {best[0]} (G2={best[1]['g2_gap']:.3f}R, Pearson={best[1]['pearson']:+.3f})")
        print(f"  → ADVANCE {best[0]} to Phase 2 backtest with weight = "
              f"{'+' if best[1]['direction'] == 'high' else '-'}0.05 × min(1, |P|/0.23)")
    else:
        print(f"\n✗ 0 features passed all gates.")
        best_g2 = max(results.items(), key=lambda x: x[1].get("g2_gap") or 0.0)
        print(f"  Best G2: {best_g2[0]} = {best_g2[1].get('g2_gap', 0):.3f}R")
        print(f"  → CLOSE Form 4 insider buy axis.")
        print(f"  → Document in memory/project_orb_gainers_lineage.md as 23rd closed axis.")
        print(f"  → Next: 13D/13G activist axis or FINRA short interest.")

    # ── Pairwise ρ summary ────────────────────────────────────────────────
    print("\n" + "=" * 72)
    print("PAIRWISE ρ MATRIX (all features vs each other)")
    print("=" * 72)
    feat_names = list(feat_arrays.keys())
    header = f"{'':30s}" + "".join(f"{n[:10]:>12s}" for n in feat_names)
    print(header)
    for i, name_i in enumerate(feat_names):
        pairs_i = feat_arrays[name_i]
        row = f"{name_i:30s}"
        for j, name_j in enumerate(feat_names):
            pairs_j = feat_arrays[name_j]
            rho = pairwise_rho(
                [p[0] for p in pairs_i], [p[1] for p in pairs_i],
                [p[0] for p in pairs_j], [p[1] for p in pairs_j],
            )
            row += f"{rho:+.2f}      " if rho is not None else f"{'n/a':>12s}"
        print(row)

    return results


if __name__ == "__main__":
    main()
