"""Research probe for SEC Form 4 insider-buy idle-alpha ideas.

This tool downloads the SEC's quarterly insider transaction flat files,
filters them to the current backtest universe, and tests simple
cluster-buy signals as standalone equal-weight curves.

It is intentionally conservative:
  - only exact Form 4 filings are used (no amendments)
  - only non-derivative P-code acquisitions are used
  - signals enter on the next trading day's open
  - signals exit on a fixed future close
"""

from __future__ import annotations

import argparse
import csv
import datetime as dt
import io
import json
import urllib.request
import zipfile
from collections import defaultdict
from dataclasses import dataclass
from pathlib import Path
from typing import Any

from apps.backtester.run import _build_merged_snapshot_store, load_manifest, resolve_config
from libs.backtest.domain import DailyPortfolioState
from libs.backtest.metrics import (
    compute_max_drawdown_pct,
    compute_sharpe_ratio,
    compute_total_return_pct,
)
from libs.common.logging import configure_logging


_DEFAULT_USER_AGENT = "fithia2-form4-probe/1.0 (local research; contact: dev@example.com)"


@dataclass(frozen=True)
class InsiderSignalSpec:
    name: str
    min_owner_count: int
    min_total_value: float
    hold_days: int
    require_officer_or_director: bool = False
    max_positions: int = 10
    max_new_per_day: int = 3


def _parse_date(value: str, *, is_end: bool = False) -> dt.date:
    parts = value.split("-")
    if len(parts) == 1 and len(value) == 4 and value.isdigit():
        year = int(value)
        return dt.date(year, 12, 31) if is_end else dt.date(year, 1, 1)
    if len(parts) == 2 and all(part.isdigit() for part in parts):
        year = int(parts[0])
        month = int(parts[1])
        if is_end:
            next_month = dt.date(year + (month // 12), (month % 12) + 1, 1)
            return next_month - dt.timedelta(days=1)
        return dt.date(year, month, 1)
    return dt.date.fromisoformat(value)


def _quarter_range(start_date: dt.date, end_date: dt.date) -> list[tuple[int, int]]:
    year = start_date.year
    quarter = (start_date.month - 1) // 3 + 1
    end_key = (end_date.year, (end_date.month - 1) // 3 + 1)
    quarters: list[tuple[int, int]] = []
    while (year, quarter) <= end_key:
        quarters.append((year, quarter))
        quarter += 1
        if quarter == 5:
            year += 1
            quarter = 1
    return quarters


def _quarter_zip_path(cache_dir: Path, year: int, quarter: int) -> Path:
    return cache_dir / f"{year}q{quarter}_form345.zip"


def _quarter_zip_url(year: int, quarter: int) -> str:
    return (
        "https://www.sec.gov/files/structureddata/data/"
        f"insider-transactions-data-sets/{year}q{quarter}_form345.zip"
    )


def _ensure_quarter_zip(
    cache_dir: Path,
    year: int,
    quarter: int,
    *,
    user_agent: str,
) -> Path:
    cache_dir.mkdir(parents=True, exist_ok=True)
    path = _quarter_zip_path(cache_dir, year, quarter)
    if path.exists() and path.stat().st_size > 0:
        return path
    request = urllib.request.Request(
        _quarter_zip_url(year, quarter),
        headers={"User-Agent": user_agent},
    )
    with urllib.request.urlopen(request, timeout=60) as response:
        data = response.read()
    path.write_bytes(data)
    return path


def _parse_sec_date(value: str | None) -> dt.date | None:
    if not value:
        return None
    try:
        return dt.datetime.strptime(value, "%d-%b-%Y").date()
    except ValueError:
        return None


def _load_form4_cluster_events(
    *,
    cache_dir: Path,
    start_date: dt.date,
    end_date: dt.date,
    allowed_symbols: set[str],
    user_agent: str,
) -> dict[tuple[dt.date, str], dict[str, Any]]:
    events: dict[tuple[dt.date, str], dict[str, Any]] = {}
    for year, quarter in _quarter_range(start_date, end_date):
        path = _ensure_quarter_zip(cache_dir, year, quarter, user_agent=user_agent)
        with zipfile.ZipFile(path) as zf:
            submissions: dict[str, dict[str, Any]] = {}
            with zf.open("SUBMISSION.tsv") as handle:
                reader = csv.DictReader(
                    io.TextIOWrapper(handle, encoding="utf-8", newline=""),
                    delimiter="\t",
                )
                for row in reader:
                    symbol = str(row.get("ISSUERTRADINGSYMBOL") or "").strip().upper()
                    if not symbol or symbol not in allowed_symbols:
                        continue
                    if str(row.get("DOCUMENT_TYPE") or "").strip().upper() != "4":
                        continue
                    filing_date = _parse_sec_date(row.get("FILING_DATE"))
                    if filing_date is None or filing_date < start_date or filing_date > end_date:
                        continue
                    submissions[str(row["ACCESSION_NUMBER"])] = {
                        "symbol": symbol,
                        "filing_date": filing_date,
                    }

            if not submissions:
                continue

            relationships: dict[str, list[tuple[str, str]]] = defaultdict(list)
            with zf.open("REPORTINGOWNER.tsv") as handle:
                reader = csv.DictReader(
                    io.TextIOWrapper(handle, encoding="utf-8", newline=""),
                    delimiter="\t",
                )
                for row in reader:
                    accession = str(row["ACCESSION_NUMBER"])
                    if accession not in submissions:
                        continue
                    owner_cik = str(row.get("RPTOWNERCIK") or "").strip()
                    relationship = str(row.get("RPTOWNER_RELATIONSHIP") or "").strip().lower()
                    relationships[accession].append((owner_cik, relationship))

            with zf.open("NONDERIV_TRANS.tsv") as handle:
                reader = csv.DictReader(
                    io.TextIOWrapper(handle, encoding="utf-8", newline=""),
                    delimiter="\t",
                )
                for row in reader:
                    accession = str(row["ACCESSION_NUMBER"])
                    submission = submissions.get(accession)
                    if submission is None:
                        continue
                    if str(row.get("TRANS_CODE") or "").strip().upper() != "P":
                        continue
                    if str(row.get("TRANS_ACQUIRED_DISP_CD") or "").strip().upper() != "A":
                        continue
                    try:
                        shares = float(row.get("TRANS_SHARES") or 0.0)
                        price = float(row.get("TRANS_PRICEPERSHARE") or 0.0)
                    except (TypeError, ValueError):
                        continue
                    if shares <= 0 or price <= 0:
                        continue

                    key = (submission["filing_date"], submission["symbol"])
                    event = events.setdefault(
                        key,
                        {
                            "filing_date": submission["filing_date"],
                            "symbol": submission["symbol"],
                            "total_value": 0.0,
                            "transaction_count": 0,
                            "owners": set(),
                            "officer_or_director": set(),
                        },
                    )
                    event["total_value"] += shares * price
                    event["transaction_count"] += 1
                    for owner_cik, relationship in relationships.get(accession, []):
                        if not owner_cik:
                            continue
                        event["owners"].add(owner_cik)
                        if "officer" in relationship or "director" in relationship:
                            event["officer_or_director"].add(owner_cik)
    return events


def _build_entries_for_spec(
    *,
    store: Any,
    events: dict[tuple[dt.date, str], dict[str, Any]],
    spec: InsiderSignalSpec,
) -> dict[dt.date, list[dict[str, Any]]]:
    trading_days = store.all_trading_days()
    trading_index = {date: idx for idx, date in enumerate(trading_days)}
    entries: dict[dt.date, list[dict[str, Any]]] = defaultdict(list)

    for event in events.values():
        owner_count = len(event["owners"])
        if owner_count < spec.min_owner_count:
            continue
        if float(event["total_value"]) < spec.min_total_value:
            continue
        if spec.require_officer_or_director and not event["officer_or_director"]:
            continue

        filing_date = event["filing_date"]
        entry_date = next((date for date in trading_days if date > filing_date), None)
        if entry_date is None:
            continue
        entry_bar = store.get_bar(event["symbol"], entry_date)
        if not entry_bar or not entry_bar.get("open") or not entry_bar.get("close"):
            continue
        entry_index = trading_index.get(entry_date)
        if entry_index is None or entry_index + spec.hold_days >= len(trading_days):
            continue
        exit_date = trading_days[entry_index + spec.hold_days]
        exit_bar = store.get_bar(event["symbol"], exit_date)
        if not exit_bar or not exit_bar.get("close"):
            continue

        entries[entry_date].append(
            {
                "symbol": event["symbol"],
                "exit_date": exit_date,
                "score": (owner_count, float(event["total_value"])),
                "owner_count": owner_count,
                "total_value": float(event["total_value"]),
            }
        )
    return entries


def _simulate_equal_weight_curve(
    *,
    store: Any,
    start_date: dt.date,
    end_date: dt.date,
    entries_by_date: dict[dt.date, list[dict[str, Any]]],
    spec: InsiderSignalSpec,
    capital: float,
) -> tuple[list[DailyPortfolioState], int]:
    trading_days = [
        date for date in store.all_trading_days() if start_date <= date <= end_date
    ]
    equity = capital
    peak = capital
    curve: list[DailyPortfolioState] = []
    open_positions: list[dict[str, Any]] = []
    trade_count = 0

    for date in trading_days:
        if date in entries_by_date and len(open_positions) < spec.max_positions:
            existing_symbols = {position["symbol"] for position in open_positions}
            ranked = sorted(
                entries_by_date[date],
                key=lambda row: (row["score"][0], row["score"][1]),
                reverse=True,
            )
            added = 0
            for row in ranked:
                if row["symbol"] in existing_symbols:
                    continue
                bar = store.get_bar(row["symbol"], date)
                if not bar or not bar.get("open"):
                    continue
                open_positions.append(
                    {
                        "symbol": row["symbol"],
                        "exit_date": row["exit_date"],
                        "prev_price": float(bar["open"]),
                    }
                )
                existing_symbols.add(row["symbol"])
                trade_count += 1
                added += 1
                if added >= spec.max_new_per_day or len(open_positions) >= spec.max_positions:
                    break

        if open_positions:
            daily_returns: list[float] = []
            updated_positions: list[dict[str, Any]] = []
            for position in open_positions:
                bar = store.get_bar(position["symbol"], date)
                if not bar or not bar.get("close"):
                    continue
                close_price = float(bar["close"])
                prev_price = float(position["prev_price"])
                if prev_price <= 0:
                    continue
                daily_returns.append(close_price / prev_price - 1.0)
                updated_positions.append(
                    {
                        "symbol": position["symbol"],
                        "exit_date": position["exit_date"],
                        "prev_price": close_price,
                    }
                )
            if daily_returns:
                equity *= 1.0 + sum(daily_returns) / len(daily_returns)
            open_positions = [
                position for position in updated_positions if position["exit_date"] > date
            ]

        peak = max(peak, equity)
        curve.append(
            DailyPortfolioState(
                date=date,
                equity=equity,
                sizing_equity=equity,
                cash_available=equity,
                gross_exposure=float(len(open_positions)) * 10.0,
                net_exposure=float(len(open_positions)) * 10.0,
                reserved_risk_budget=0.0,
                unrealized_pnl=0.0,
                realized_pnl=0.0,
                open_positions=[position["symbol"] for position in open_positions],
                daily_new_risk_used=0.0,
                peak_equity=peak,
                current_drawdown_pct=((peak - equity) / peak * 100.0) if peak > 0 else 0.0,
            )
        )
    return curve, trade_count


def _yearly_return_map(curve: list[DailyPortfolioState]) -> dict[str, float]:
    if not curve:
        return {}
    year_start: dict[int, float] = {}
    year_end: dict[int, float] = {}
    for state in curve:
        year_start.setdefault(state.date.year, state.equity)
        year_end[state.date.year] = state.equity
    return {
        str(year): round((year_end[year] / year_start[year] - 1.0) * 100.0, 2)
        for year in sorted(year_start)
        if year_start[year] > 0
    }


def _default_specs() -> list[InsiderSignalSpec]:
    return [
        InsiderSignalSpec("owners2_value500k_hold5", 2, 500_000.0, 5),
        InsiderSignalSpec("owners2_value1m_hold5", 2, 1_000_000.0, 5),
        InsiderSignalSpec("owners2_value500k_hold10", 2, 500_000.0, 10),
        InsiderSignalSpec("owners2_value500k_hold5_offdir", 2, 500_000.0, 5, True),
        InsiderSignalSpec("owners1_value1m_hold5", 1, 1_000_000.0, 5),
    ]


def _run_probe(args: argparse.Namespace) -> list[dict[str, Any]]:
    start_date = _parse_date(args.start)
    end_date = _parse_date(args.end, is_end=True)
    rows: list[dict[str, Any]] = []

    for config_path in args.config:
        manifest = load_manifest(config_path)
        config = resolve_config(manifest)
        store = _build_merged_snapshot_store(
            manifest,
            config,
            snapshot_dir_override=None,
        ).slice_by_date_range(start_date, end_date)
        allowed_symbols = {str(symbol).upper() for symbol in store._bars.keys()}
        events = _load_form4_cluster_events(
            cache_dir=Path(args.cache_dir),
            start_date=start_date,
            end_date=end_date,
            allowed_symbols=allowed_symbols,
            user_agent=args.user_agent,
        )
        specs = _default_specs()
        for spec in specs:
            entries = _build_entries_for_spec(store=store, events=events, spec=spec)
            curve, trade_count = _simulate_equal_weight_curve(
                store=store,
                start_date=start_date,
                end_date=end_date,
                entries_by_date=entries,
                spec=spec,
                capital=args.capital,
            )
            rows.append(
                {
                    "config": config_path,
                    "spec": spec.name,
                    "signals": int(sum(len(value) for value in entries.values())),
                    "trades": trade_count,
                    "total_return_pct": round(compute_total_return_pct(curve) or 0.0, 2),
                    "max_drawdown_pct": round(compute_max_drawdown_pct(curve) or 0.0, 2),
                    "sharpe_ratio": round(compute_sharpe_ratio(curve) or 0.0, 3),
                    "yearly_return_pct": _yearly_return_map(curve),
                }
            )
    return rows


def main() -> None:
    parser = argparse.ArgumentParser(description="Probe SEC Form 4 insider-buy strategies")
    parser.add_argument("--config", action="append", required=True)
    parser.add_argument("--start", required=True)
    parser.add_argument("--end", required=True)
    parser.add_argument("--capital", type=float, default=10_000.0)
    parser.add_argument("--cache-dir", default="data/cache/sec_form345")
    parser.add_argument("--user-agent", default=_DEFAULT_USER_AGENT)
    parser.add_argument("--json", action="store_true")
    args = parser.parse_args()
    configure_logging("WARNING")
    rows = _run_probe(args)
    if args.json:
        print(json.dumps(rows, indent=2))
    else:
        for row in rows:
            print(row)


if __name__ == "__main__":
    main()