fithia2/apps/tools/deepeval_effective_n.py

#!/usr/bin/env python3
"""DeepEval Phase 2.2: Effective Independent Sample Count calculator.

Reads engine attribution and trade blotter data from a backtest run to compute
per-engine effective_n — the number of truly independent trade observations,
accounting for same-ticker repeated events.

Usage:
    python -m apps.tools.deepeval_effective_n \
        --run-dir runs/v1045_wfv/walk_forward/fold_07/train/bt_*

    Or for all folds:
    python -m apps.tools.deepeval_effective_n \
        --wfv-dir runs/v1045_wfv
"""
from __future__ import annotations

import argparse
import csv
import json
from collections import defaultdict
from datetime import date, timedelta
from pathlib import Path

MIN_GAP_DAYS = 126  # ~6 months between events for independence
MAX_PER_TICKER = 3  # cap independent observations per ticker


def load_trade_blotter(run_dir: Path) -> list[dict]:
    """Load trade blotter from parquet or CSV."""
    parquet_path = run_dir / "artifacts" / "trade_blotter.parquet"
    if parquet_path.exists():
        try:
            import pyarrow.parquet as pq
            table = pq.read_table(parquet_path)
            return table.to_pylist()
        except ImportError:
            pass

    csv_path = run_dir / "artifacts" / "trade_blotter.csv"
    if csv_path.exists():
        with open(csv_path) as f:
            return list(csv.DictReader(f))
    return []


def load_attribution(run_dir: Path) -> dict[str, dict]:
    """Load engine attribution CSV."""
    csv_path = run_dir / "metrics" / "attribution_by_engine.csv"
    if not csv_path.exists():
        return {}
    result = {}
    with open(csv_path) as f:
        for row in csv.DictReader(f):
            result[row["engine_id"]] = row
    return result


def compute_effective_n(trades: list[dict], min_gap_days: int = MIN_GAP_DAYS) -> dict:
    """Compute effective independent sample count for trades grouped by engine.

    Returns dict of engine_id -> {
        total_trades, unique_tickers, effective_n,
        ticker_breakdown: {ticker: {total, independent}}
    }
    """
    # Group trades by engine_id
    by_engine: dict[str, list[dict]] = defaultdict(list)
    for t in trades:
        eid = t.get("engine_id", "default")
        by_engine[eid].append(t)

    results = {}
    for engine_id, engine_trades in by_engine.items():
        # Group by ticker
        by_ticker: dict[str, list[date]] = defaultdict(list)
        for t in engine_trades:
            symbol = t.get("symbol", "")
            entry_str = t.get("entry_date", "")
            if isinstance(entry_str, str) and entry_str:
                try:
                    entry_date = date.fromisoformat(entry_str[:10])
                except ValueError:
                    continue
            elif isinstance(entry_str, date):
                entry_date = entry_str
            else:
                continue
            by_ticker[symbol].append(entry_date)

        ticker_breakdown = {}
        effective_n = 0
        for ticker, dates in by_ticker.items():
            sorted_dates = sorted(dates)
            independent = 1  # first event always counts
            last_counted = sorted_dates[0]
            for d in sorted_dates[1:]:
                if (d - last_counted).days >= min_gap_days:
                    independent += 1
                    last_counted = d
            capped = min(independent, MAX_PER_TICKER)
            effective_n += capped
            ticker_breakdown[ticker] = {
                "total": len(dates),
                "independent": independent,
                "capped": capped,
            }

        results[engine_id] = {
            "total_trades": len(engine_trades),
            "unique_tickers": len(by_ticker),
            "effective_n": effective_n,
            "ticker_breakdown": ticker_breakdown,
        }

    return results


def classify(effective_n: int) -> str:
    if effective_n < 3:
        return "shadow_only"
    elif effective_n < 8:
        return "reduced_budget"
    else:
        return "full_budget"


def print_report(engine_stats: dict, attribution: dict) -> None:
    """Print formatted report."""
    print(f"\n{'Engine':<50} {'Trades':>6} {'Tickers':>7} {'Eff-N':>5} {'Class':<15} {'PnL':>10}")
    print("-" * 100)

    # Sort by effective_n ascending (worst first)
    sorted_engines = sorted(engine_stats.items(), key=lambda x: x[1]["effective_n"])

    shadow_count = 0
    reduced_count = 0
    full_count = 0

    for eid, stats in sorted_engines:
        cls = classify(stats["effective_n"])
        if cls == "shadow_only":
            shadow_count += 1
        elif cls == "reduced_budget":
            reduced_count += 1
        else:
            full_count += 1

        pnl = ""
        if eid in attribution:
            pnl = f"${float(attribution[eid].get('net_pnl', 0)):>9,.0f}"

        is_exact = "exact" in eid
        marker = " *" if is_exact else ""
        print(
            f"{eid[:48] + marker:<50} "
            f"{stats['total_trades']:>6} "
            f"{stats['unique_tickers']:>7} "
            f"{stats['effective_n']:>5} "
            f"{cls:<15} "
            f"{pnl:>10}"
        )

    print("-" * 100)
    print(f"Shadow-only (eff_n < 3): {shadow_count}")
    print(f"Reduced budget (3 <= eff_n < 8): {reduced_count}")
    print(f"Full budget (eff_n >= 8): {full_count}")
    print(f"Total engines with trades: {len(engine_stats)}")
    print("(* = exact pocket engine)")


def process_run_dir(run_dir: Path) -> dict:
    """Process a single run directory."""
    trades = load_trade_blotter(run_dir)
    if not trades:
        print(f"No trades found in {run_dir}")
        return {}
    attribution = load_attribution(run_dir)
    return compute_effective_n(trades)


def main() -> None:
    parser = argparse.ArgumentParser(description="DeepEval Phase 2.2: Effective-N Calculator")
    parser.add_argument("--run-dir", help="Single backtest run directory")
    parser.add_argument("--wfv-dir", help="Walk-forward validation directory (processes all folds)")
    parser.add_argument("--output", help="Save JSON report to file")
    args = parser.parse_args()

    if args.wfv_dir:
        # Aggregate across all train folds
        wfv_dir = Path(args.wfv_dir)
        all_trades = []
        for fold_dir in sorted(wfv_dir.glob("walk_forward/fold_*/train/bt_*")):
            trades = load_trade_blotter(fold_dir)
            all_trades.extend(trades)
        if not all_trades:
            print("No trades found in WFV folds")
            return

        print(f"Loaded {len(all_trades)} trades from WFV train folds")
        engine_stats = compute_effective_n(all_trades)

        # Load attribution from latest fold for PnL reference
        latest_fold = sorted(wfv_dir.glob("walk_forward/fold_*/train/bt_*"))[-1] if all_trades else None
        attribution = load_attribution(latest_fold) if latest_fold else {}

    elif args.run_dir:
        run_dir = Path(args.run_dir)
        trades = load_trade_blotter(run_dir)
        if not trades:
            print(f"No trades found in {run_dir}")
            return
        engine_stats = compute_effective_n(trades)
        attribution = load_attribution(run_dir)
    else:
        parser.error("Provide --run-dir or --wfv-dir")
        return

    print_report(engine_stats, attribution)

    if args.output:
        # Prepare serializable report
        report = {}
        for eid, stats in engine_stats.items():
            report[eid] = {
                **stats,
                "classification": classify(stats["effective_n"]),
                "is_exact": "exact" in eid,
            }
        with open(args.output, "w") as f:
            json.dump(report, f, indent=2)
        print(f"\nJSON report saved to {args.output}")


if __name__ == "__main__":
    main()