"""Build leakage-safe daily Form 4 cluster parquet from SEC flat files. The output is a daily same-day cluster cache keyed by filing date. Runtime code uses this cache conservatively on the next trading day only. """ from __future__ import annotations import argparse import csv import datetime as dt import io import math import urllib.error import urllib.request import zipfile from collections import defaultdict from dataclasses import dataclass from pathlib import Path from typing import Any import pyarrow as pa import pyarrow.parquet as pq from apps.backtester.run import _build_merged_snapshot_store, load_manifest, resolve_config from libs.common.logging import configure_logging _DEFAULT_USER_AGENT = "fithia2-form4-cache/1.0 (local research; contact: dev@example.com)" @dataclass(frozen=True) class RawForm4Transaction: symbol: str filing_date: dt.date transaction_date: dt.date owner_cik: str owner_name: str owner_relationship: str owner_title: str owner_text: str shares: float price: float total_value: float shares_owned_following: float purchase_pct_of_holding: float is_officer: bool is_director: bool is_ten_percent_owner: bool is_other: bool is_ceo: bool is_cfo: bool is_president: bool is_chair: bool is_c_suite: bool def _parse_date(value: str, *, is_end: bool = False) -> dt.date: parts = value.split("-") if len(parts) == 1 and len(value) == 4 and value.isdigit(): year = int(value) return dt.date(year, 12, 31) if is_end else dt.date(year, 1, 1) if len(parts) == 2 and all(part.isdigit() for part in parts): year = int(parts[0]) month = int(parts[1]) if is_end: next_month = dt.date(year + (month // 12), (month % 12) + 1, 1) return next_month - dt.timedelta(days=1) return dt.date(year, month, 1) return dt.date.fromisoformat(value) def _quarter_range(start_date: dt.date, end_date: dt.date) -> list[tuple[int, int]]: year = start_date.year quarter = (start_date.month - 1) // 3 + 1 end_key = (end_date.year, (end_date.month - 1) // 3 + 1) quarters: list[tuple[int, int]] = [] while (year, quarter) <= end_key: quarters.append((year, quarter)) quarter += 1 if quarter == 5: year += 1 quarter = 1 return quarters def _quarter_zip_path(cache_dir: Path, year: int, quarter: int) -> Path: return cache_dir / f"{year}q{quarter}_form345.zip" def _quarter_zip_url(year: int, quarter: int) -> str: return ( "https://www.sec.gov/files/structureddata/data/" f"insider-transactions-data-sets/{year}q{quarter}_form345.zip" ) def _ensure_quarter_zip(cache_dir: Path, year: int, quarter: int, *, user_agent: str) -> Path | None: cache_dir.mkdir(parents=True, exist_ok=True) path = _quarter_zip_path(cache_dir, year, quarter) if path.exists() and path.stat().st_size > 0: return path request = urllib.request.Request(_quarter_zip_url(year, quarter), headers={"User-Agent": user_agent}) try: with urllib.request.urlopen(request, timeout=90) as response: data = response.read() except urllib.error.HTTPError as exc: if exc.code == 404: return None raise path.write_bytes(data) return path def _parse_sec_date(value: str | None) -> dt.date | None: if not value: return None try: return dt.datetime.strptime(value, "%d-%b-%Y").date() except ValueError: return None def _coerce_float(value: Any) -> float | None: try: result = float(value) except (TypeError, ValueError): return None if math.isnan(result) or math.isinf(result): return None return result def _is_officer_or_director(relationship: str, title: str) -> bool: combined = f"{relationship} {title}".strip().lower() return any(token in combined for token in ("director", "officer", "chief", "ceo", "cfo", "coo", "president", "chair")) def _owner_role_flags(relationship: str, title: str, text: str) -> dict[str, bool]: relationship_norm = str(relationship or "").strip().lower().replace(" ", "") combined = " ".join( part.strip().lower() for part in (relationship, title, text) if str(part or "").strip() ) padded = f" {combined} " is_officer = "officer" in relationship_norm is_director = "director" in relationship_norm is_ten_percent_owner = "tenpercentowner" in relationship_norm or "10% owner" in combined is_other = "other" in relationship_norm is_ceo = "chief executive officer" in combined or " ceo " in padded is_cfo = "chief financial officer" in combined or " cfo " in padded is_president = "president" in combined is_chair = any(token in combined for token in ("chairman", "chairwoman", "chairperson", "chair ")) is_cao = "chief accounting officer" in combined is_coo = "chief operating officer" in combined or " coo " in padded return { "is_officer": is_officer, "is_director": is_director, "is_ten_percent_owner": is_ten_percent_owner, "is_other": is_other, "is_ceo": is_ceo, "is_cfo": is_cfo, "is_president": is_president, "is_chair": is_chair, "is_c_suite": bool(is_ceo or is_cfo or is_cao or is_coo), } def _owner_role_weight(flags: dict[str, bool]) -> float: weight = 0.0 if flags.get("is_cfo"): weight = max(weight, 3.0) if flags.get("is_ceo"): weight = max(weight, 2.5) if flags.get("is_c_suite"): weight = max(weight, 2.0) if flags.get("is_chair") or flags.get("is_president"): weight = max(weight, 1.5) if flags.get("is_officer") or flags.get("is_director"): weight = max(weight, 1.0) if flags.get("is_ten_percent_owner"): weight = max(weight, 0.5) if flags.get("is_other"): weight = max(weight, 0.25) return weight def _load_form4_transactions( *, cache_dir: Path, start_date: dt.date, end_date: dt.date, allowed_symbols: set[str], user_agent: str, ) -> tuple[list[RawForm4Transaction], list[str]]: transactions: list[RawForm4Transaction] = [] skipped_quarters: list[str] = [] for year, quarter in _quarter_range(start_date, end_date): path = _ensure_quarter_zip(cache_dir, year, quarter, user_agent=user_agent) if path is None: skipped_quarters.append(f"{year}Q{quarter}") continue with zipfile.ZipFile(path) as zf: submissions: dict[str, dict[str, Any]] = {} with zf.open("SUBMISSION.tsv") as handle: reader = csv.DictReader(io.TextIOWrapper(handle, encoding="utf-8", newline=""), delimiter="\t") for row in reader: symbol = str(row.get("ISSUERTRADINGSYMBOL") or "").strip().upper() if not symbol or symbol not in allowed_symbols: continue if str(row.get("DOCUMENT_TYPE") or "").strip().upper() != "4": continue filing_date = _parse_sec_date(row.get("FILING_DATE")) if filing_date is None or filing_date < start_date or filing_date > end_date: continue submissions[str(row["ACCESSION_NUMBER"])] = { "symbol": symbol, "filing_date": filing_date, } if not submissions: continue relationships: dict[str, list[tuple[str, str, str, str, str, dict[str, bool]]]] = defaultdict(list) with zf.open("REPORTINGOWNER.tsv") as handle: reader = csv.DictReader(io.TextIOWrapper(handle, encoding="utf-8", newline=""), delimiter="\t") for row in reader: accession = str(row["ACCESSION_NUMBER"]) if accession not in submissions: continue owner_cik = str(row.get("RPTOWNERCIK") or "").strip() owner_name = str(row.get("RPTOWNERNAME") or "").strip() relationship = str(row.get("RPTOWNER_RELATIONSHIP") or "").strip() title = str(row.get("RPTOWNER_TITLE") or "").strip() text = str(row.get("RPTOWNER_TXT") or "").strip() relationships[accession].append( (owner_cik, owner_name, relationship, title, text, _owner_role_flags(relationship, title, text)) ) with zf.open("NONDERIV_TRANS.tsv") as handle: reader = csv.DictReader(io.TextIOWrapper(handle, encoding="utf-8", newline=""), delimiter="\t") for row in reader: accession = str(row["ACCESSION_NUMBER"]) submission = submissions.get(accession) if submission is None: continue transaction_code = str(row.get("TRANS_CODE") or "").strip().upper() if transaction_code != "P": continue shares = _coerce_float(row.get("TRANS_SHARES")) price = _coerce_float(row.get("TRANS_PRICEPERSHARE")) if shares is None or price is None or shares <= 0 or price <= 0: continue transaction_date = _parse_sec_date(row.get("TRANS_DATE")) or submission["filing_date"] shares_owned_following = _coerce_float(row.get("SHRS_OWND_FOLWNG_TRANS")) or 0.0 purchase_pct = 0.0 if shares_owned_following > 0: purchase_pct = min(1.0, shares / shares_owned_following) rels = relationships.get(accession) or [("", "", "", "", "", _owner_role_flags("", "", ""))] for owner_cik, owner_name, relationship, title, text, flags in rels: transactions.append( RawForm4Transaction( symbol=submission["symbol"], filing_date=submission["filing_date"], transaction_date=transaction_date, owner_cik=owner_cik, owner_name=owner_name, owner_relationship=relationship, owner_title=title, owner_text=text, shares=shares, price=price, total_value=shares * price, shares_owned_following=shares_owned_following, purchase_pct_of_holding=purchase_pct, is_officer=bool(flags["is_officer"]), is_director=bool(flags["is_director"]), is_ten_percent_owner=bool(flags["is_ten_percent_owner"]), is_other=bool(flags["is_other"]), is_ceo=bool(flags["is_ceo"]), is_cfo=bool(flags["is_cfo"]), is_president=bool(flags["is_president"]), is_chair=bool(flags["is_chair"]), is_c_suite=bool(flags["is_c_suite"]), ) ) return transactions, skipped_quarters def _aggregate_daily_events(transactions: list[RawForm4Transaction]) -> list[dict[str, Any]]: grouped: dict[tuple[dt.date, str], list[RawForm4Transaction]] = defaultdict(list) for row in transactions: grouped[(row.filing_date, row.symbol)].append(row) events: list[dict[str, Any]] = [] for (filing_date, symbol), rows in grouped.items(): owner_role_map: dict[str, dict[str, bool]] = {} owner_keys: set[str] = set() for idx, row in enumerate(rows): owner_key = row.owner_cik or row.owner_name or f"{symbol}:{idx}" owner_keys.add(owner_key) flags = owner_role_map.setdefault( owner_key, { "is_officer": False, "is_director": False, "is_ten_percent_owner": False, "is_other": False, "is_ceo": False, "is_cfo": False, "is_president": False, "is_chair": False, "is_c_suite": False, }, ) flags["is_officer"] = flags["is_officer"] or row.is_officer flags["is_director"] = flags["is_director"] or row.is_director flags["is_ten_percent_owner"] = flags["is_ten_percent_owner"] or row.is_ten_percent_owner flags["is_other"] = flags["is_other"] or row.is_other flags["is_ceo"] = flags["is_ceo"] or row.is_ceo flags["is_cfo"] = flags["is_cfo"] or row.is_cfo flags["is_president"] = flags["is_president"] or row.is_president flags["is_chair"] = flags["is_chair"] or row.is_chair flags["is_c_suite"] = flags["is_c_suite"] or row.is_c_suite purchase_values = [row.purchase_pct_of_holding for row in rows if row.purchase_pct_of_holding > 0] lag_values = [(row.filing_date - row.transaction_date).days for row in rows] transaction_dates = [row.transaction_date for row in rows] transaction_span_days = ( max((max(transaction_dates) - min(transaction_dates)).days, 0) if transaction_dates else 0 ) officer_count = sum(1 for flags in owner_role_map.values() if flags["is_officer"]) director_count = sum(1 for flags in owner_role_map.values() if flags["is_director"]) ten_percent_owner_count = sum(1 for flags in owner_role_map.values() if flags["is_ten_percent_owner"]) other_role_count = sum(1 for flags in owner_role_map.values() if flags["is_other"]) ceo_count = sum(1 for flags in owner_role_map.values() if flags["is_ceo"]) cfo_count = sum(1 for flags in owner_role_map.values() if flags["is_cfo"]) c_suite_count = sum(1 for flags in owner_role_map.values() if flags["is_c_suite"]) role_weight_score = round(sum(_owner_role_weight(flags) for flags in owner_role_map.values()), 3) events.append( { "symbol": symbol, "filing_date": filing_date.isoformat(), "as_of_date": filing_date.isoformat(), "total_value": round(sum(row.total_value for row in rows), 2), "owner_count": len(owner_keys), "transaction_count": len(rows), "event_day_count": 1, "max_purchase_pct": max(purchase_values) if purchase_values else 0.0, "median_purchase_pct": sorted(purchase_values)[len(purchase_values) // 2] if purchase_values else 0.0, "weighted_purchase_pct": max(purchase_values) if purchase_values else 0.0, "max_lag_days": max(lag_values) if lag_values else None, "min_lag_days": min(lag_values) if lag_values else None, "transaction_span_days": transaction_span_days, "officer_count": officer_count, "director_count": director_count, "ten_percent_owner_count": ten_percent_owner_count, "other_role_count": other_role_count, "ceo_count": ceo_count, "cfo_count": cfo_count, "c_suite_count": c_suite_count, "role_weight_score": role_weight_score, "has_officer_or_director": any( _is_officer_or_director(row.owner_relationship, row.owner_title) for row in rows ), } ) events.sort(key=lambda row: (row["filing_date"], row["symbol"])) return events def _allowed_symbols_from_manifest(manifest_path: str, start_date: dt.date, end_date: dt.date) -> set[str]: manifest = load_manifest(manifest_path) config = resolve_config(manifest) store = _build_merged_snapshot_store( manifest, config, snapshot_dir_override=None, ).slice_by_date_range(start_date, end_date) return {str(symbol).upper() for symbol in store._bars.keys()} def main() -> None: parser = argparse.ArgumentParser(description="Build daily Form 4 same-day cluster parquet") parser.add_argument("--config", required=True, help="Experiment manifest JSON path") parser.add_argument("--start", required=True, help="Start date (YYYY, YYYY-MM, YYYY-MM-DD)") parser.add_argument("--end", required=True, help="End date (YYYY, YYYY-MM, YYYY-MM-DD)") parser.add_argument("--cache-dir", default="data/cache/form4", help="SEC zip cache dir") parser.add_argument( "--output", default="data/reference/form4_daily_events_pit.parquet", help="Output parquet path. Keep the default for frozen v1 presets; use a separate path such as data/reference/form4_daily_events_pit_v3.parquet for experimental variants.", ) parser.add_argument("--user-agent", default=_DEFAULT_USER_AGENT, help="SEC User-Agent header") args = parser.parse_args() configure_logging("INFO") start_date = _parse_date(args.start) end_date = _parse_date(args.end, is_end=True) allowed_symbols = _allowed_symbols_from_manifest(args.config, start_date, end_date) transactions, skipped_quarters = _load_form4_transactions( cache_dir=Path(args.cache_dir), start_date=start_date, end_date=end_date, allowed_symbols=allowed_symbols, user_agent=args.user_agent, ) events = _aggregate_daily_events(transactions) output_path = Path(args.output) output_path.parent.mkdir(parents=True, exist_ok=True) table = pa.Table.from_pylist(events) pq.write_table(table, str(output_path)) print( { "output": str(output_path), "symbols": len({row["symbol"] for row in events}), "events": len(events), "transactions": len(transactions), "skipped_quarters": skipped_quarters, } ) if __name__ == "__main__": main()