Add Form4 residual-cash sleeve and UI support
parent
86d55e01f9
commit
72681e69e5
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,299 @@
|
|||||||
|
"""Build leakage-safe daily Form 4 cluster parquet from SEC flat files.
|
||||||
|
|
||||||
|
The output is a daily same-day cluster cache keyed by filing date. Runtime code
|
||||||
|
uses this cache conservatively on the next trading day only.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import csv
|
||||||
|
import datetime as dt
|
||||||
|
import io
|
||||||
|
import math
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
import zipfile
|
||||||
|
from collections import defaultdict
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import pyarrow as pa
|
||||||
|
import pyarrow.parquet as pq
|
||||||
|
|
||||||
|
from apps.backtester.run import _build_merged_snapshot_store, load_manifest, resolve_config
|
||||||
|
from libs.common.logging import configure_logging
|
||||||
|
|
||||||
|
_DEFAULT_USER_AGENT = "fithia2-form4-cache/1.0 (local research; contact: dev@example.com)"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class RawForm4Transaction:
|
||||||
|
symbol: str
|
||||||
|
filing_date: dt.date
|
||||||
|
transaction_date: dt.date
|
||||||
|
owner_cik: str
|
||||||
|
owner_relationship: str
|
||||||
|
owner_title: str
|
||||||
|
shares: float
|
||||||
|
price: float
|
||||||
|
total_value: float
|
||||||
|
shares_owned_following: float
|
||||||
|
purchase_pct_of_holding: float
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_date(value: str, *, is_end: bool = False) -> dt.date:
|
||||||
|
parts = value.split("-")
|
||||||
|
if len(parts) == 1 and len(value) == 4 and value.isdigit():
|
||||||
|
year = int(value)
|
||||||
|
return dt.date(year, 12, 31) if is_end else dt.date(year, 1, 1)
|
||||||
|
if len(parts) == 2 and all(part.isdigit() for part in parts):
|
||||||
|
year = int(parts[0])
|
||||||
|
month = int(parts[1])
|
||||||
|
if is_end:
|
||||||
|
next_month = dt.date(year + (month // 12), (month % 12) + 1, 1)
|
||||||
|
return next_month - dt.timedelta(days=1)
|
||||||
|
return dt.date(year, month, 1)
|
||||||
|
return dt.date.fromisoformat(value)
|
||||||
|
|
||||||
|
|
||||||
|
def _quarter_range(start_date: dt.date, end_date: dt.date) -> list[tuple[int, int]]:
|
||||||
|
year = start_date.year
|
||||||
|
quarter = (start_date.month - 1) // 3 + 1
|
||||||
|
end_key = (end_date.year, (end_date.month - 1) // 3 + 1)
|
||||||
|
quarters: list[tuple[int, int]] = []
|
||||||
|
while (year, quarter) <= end_key:
|
||||||
|
quarters.append((year, quarter))
|
||||||
|
quarter += 1
|
||||||
|
if quarter == 5:
|
||||||
|
year += 1
|
||||||
|
quarter = 1
|
||||||
|
return quarters
|
||||||
|
|
||||||
|
|
||||||
|
def _quarter_zip_path(cache_dir: Path, year: int, quarter: int) -> Path:
|
||||||
|
return cache_dir / f"{year}q{quarter}_form345.zip"
|
||||||
|
|
||||||
|
|
||||||
|
def _quarter_zip_url(year: int, quarter: int) -> str:
|
||||||
|
return (
|
||||||
|
"https://www.sec.gov/files/structureddata/data/"
|
||||||
|
f"insider-transactions-data-sets/{year}q{quarter}_form345.zip"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_quarter_zip(cache_dir: Path, year: int, quarter: int, *, user_agent: str) -> Path | None:
|
||||||
|
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
path = _quarter_zip_path(cache_dir, year, quarter)
|
||||||
|
if path.exists() and path.stat().st_size > 0:
|
||||||
|
return path
|
||||||
|
request = urllib.request.Request(_quarter_zip_url(year, quarter), headers={"User-Agent": user_agent})
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(request, timeout=90) as response:
|
||||||
|
data = response.read()
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
if exc.code == 404:
|
||||||
|
return None
|
||||||
|
raise
|
||||||
|
path.write_bytes(data)
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_sec_date(value: str | None) -> dt.date | None:
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return dt.datetime.strptime(value, "%d-%b-%Y").date()
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _coerce_float(value: Any) -> float | None:
|
||||||
|
try:
|
||||||
|
result = float(value)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
if math.isnan(result) or math.isinf(result):
|
||||||
|
return None
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _is_officer_or_director(relationship: str, title: str) -> bool:
|
||||||
|
combined = f"{relationship} {title}".strip().lower()
|
||||||
|
return any(token in combined for token in ("director", "officer", "chief", "ceo", "cfo", "coo", "president", "chair"))
|
||||||
|
|
||||||
|
|
||||||
|
def _load_form4_transactions(
|
||||||
|
*,
|
||||||
|
cache_dir: Path,
|
||||||
|
start_date: dt.date,
|
||||||
|
end_date: dt.date,
|
||||||
|
allowed_symbols: set[str],
|
||||||
|
user_agent: str,
|
||||||
|
) -> tuple[list[RawForm4Transaction], list[str]]:
|
||||||
|
transactions: list[RawForm4Transaction] = []
|
||||||
|
skipped_quarters: list[str] = []
|
||||||
|
|
||||||
|
for year, quarter in _quarter_range(start_date, end_date):
|
||||||
|
path = _ensure_quarter_zip(cache_dir, year, quarter, user_agent=user_agent)
|
||||||
|
if path is None:
|
||||||
|
skipped_quarters.append(f"{year}Q{quarter}")
|
||||||
|
continue
|
||||||
|
with zipfile.ZipFile(path) as zf:
|
||||||
|
submissions: dict[str, dict[str, Any]] = {}
|
||||||
|
with zf.open("SUBMISSION.tsv") as handle:
|
||||||
|
reader = csv.DictReader(io.TextIOWrapper(handle, encoding="utf-8", newline=""), delimiter="\t")
|
||||||
|
for row in reader:
|
||||||
|
symbol = str(row.get("ISSUERTRADINGSYMBOL") or "").strip().upper()
|
||||||
|
if not symbol or symbol not in allowed_symbols:
|
||||||
|
continue
|
||||||
|
if str(row.get("DOCUMENT_TYPE") or "").strip().upper() != "4":
|
||||||
|
continue
|
||||||
|
filing_date = _parse_sec_date(row.get("FILING_DATE"))
|
||||||
|
if filing_date is None or filing_date < start_date or filing_date > end_date:
|
||||||
|
continue
|
||||||
|
submissions[str(row["ACCESSION_NUMBER"])] = {
|
||||||
|
"symbol": symbol,
|
||||||
|
"filing_date": filing_date,
|
||||||
|
}
|
||||||
|
|
||||||
|
if not submissions:
|
||||||
|
continue
|
||||||
|
|
||||||
|
relationships: dict[str, list[tuple[str, str, str]]] = defaultdict(list)
|
||||||
|
with zf.open("REPORTINGOWNER.tsv") as handle:
|
||||||
|
reader = csv.DictReader(io.TextIOWrapper(handle, encoding="utf-8", newline=""), delimiter="\t")
|
||||||
|
for row in reader:
|
||||||
|
accession = str(row["ACCESSION_NUMBER"])
|
||||||
|
if accession not in submissions:
|
||||||
|
continue
|
||||||
|
owner_cik = str(row.get("RPTOWNERCIK") or "").strip()
|
||||||
|
relationship = str(row.get("OFFICER_TITLE") or "").strip()
|
||||||
|
title = str(row.get("OTHER_TEXT") or "").strip()
|
||||||
|
relationships[accession].append((owner_cik, relationship, title))
|
||||||
|
|
||||||
|
with zf.open("NONDERIV_TRANS.tsv") as handle:
|
||||||
|
reader = csv.DictReader(io.TextIOWrapper(handle, encoding="utf-8", newline=""), delimiter="\t")
|
||||||
|
for row in reader:
|
||||||
|
accession = str(row["ACCESSION_NUMBER"])
|
||||||
|
submission = submissions.get(accession)
|
||||||
|
if submission is None:
|
||||||
|
continue
|
||||||
|
transaction_code = str(row.get("TRANS_CODE") or "").strip().upper()
|
||||||
|
if transaction_code != "P":
|
||||||
|
continue
|
||||||
|
shares = _coerce_float(row.get("TRANS_SHARES"))
|
||||||
|
price = _coerce_float(row.get("TRANS_PRICEPERSHARE"))
|
||||||
|
if shares is None or price is None or shares <= 0 or price <= 0:
|
||||||
|
continue
|
||||||
|
transaction_date = _parse_sec_date(row.get("TRANS_DATE")) or submission["filing_date"]
|
||||||
|
shares_owned_following = _coerce_float(row.get("SHRS_OWND_FOLWNG_TRANS")) or 0.0
|
||||||
|
purchase_pct = 0.0
|
||||||
|
if shares_owned_following > 0:
|
||||||
|
purchase_pct = min(1.0, shares / shares_owned_following)
|
||||||
|
rels = relationships.get(accession) or [("", "", "")]
|
||||||
|
for owner_cik, relationship, title in rels:
|
||||||
|
transactions.append(
|
||||||
|
RawForm4Transaction(
|
||||||
|
symbol=submission["symbol"],
|
||||||
|
filing_date=submission["filing_date"],
|
||||||
|
transaction_date=transaction_date,
|
||||||
|
owner_cik=owner_cik,
|
||||||
|
owner_relationship=relationship,
|
||||||
|
owner_title=title,
|
||||||
|
shares=shares,
|
||||||
|
price=price,
|
||||||
|
total_value=shares * price,
|
||||||
|
shares_owned_following=shares_owned_following,
|
||||||
|
purchase_pct_of_holding=purchase_pct,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
return transactions, skipped_quarters
|
||||||
|
|
||||||
|
|
||||||
|
def _aggregate_daily_events(transactions: list[RawForm4Transaction]) -> list[dict[str, Any]]:
|
||||||
|
grouped: dict[tuple[dt.date, str], list[RawForm4Transaction]] = defaultdict(list)
|
||||||
|
for row in transactions:
|
||||||
|
grouped[(row.filing_date, row.symbol)].append(row)
|
||||||
|
|
||||||
|
events: list[dict[str, Any]] = []
|
||||||
|
for (filing_date, symbol), rows in grouped.items():
|
||||||
|
owner_keys = {row.owner_cik or f"{symbol}:{idx}" for idx, row in enumerate(rows)}
|
||||||
|
purchase_values = [row.purchase_pct_of_holding for row in rows if row.purchase_pct_of_holding > 0]
|
||||||
|
lag_values = [(row.filing_date - row.transaction_date).days for row in rows]
|
||||||
|
events.append(
|
||||||
|
{
|
||||||
|
"symbol": symbol,
|
||||||
|
"filing_date": filing_date.isoformat(),
|
||||||
|
"as_of_date": filing_date.isoformat(),
|
||||||
|
"total_value": round(sum(row.total_value for row in rows), 2),
|
||||||
|
"owner_count": len(owner_keys),
|
||||||
|
"transaction_count": len(rows),
|
||||||
|
"event_day_count": 1,
|
||||||
|
"max_purchase_pct": max(purchase_values) if purchase_values else 0.0,
|
||||||
|
"median_purchase_pct": sorted(purchase_values)[len(purchase_values) // 2] if purchase_values else 0.0,
|
||||||
|
"weighted_purchase_pct": max(purchase_values) if purchase_values else 0.0,
|
||||||
|
"max_lag_days": max(lag_values) if lag_values else None,
|
||||||
|
"min_lag_days": min(lag_values) if lag_values else None,
|
||||||
|
"has_officer_or_director": any(
|
||||||
|
_is_officer_or_director(row.owner_relationship, row.owner_title)
|
||||||
|
for row in rows
|
||||||
|
),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
events.sort(key=lambda row: (row["filing_date"], row["symbol"]))
|
||||||
|
return events
|
||||||
|
|
||||||
|
|
||||||
|
def _allowed_symbols_from_manifest(manifest_path: str, start_date: dt.date, end_date: dt.date) -> set[str]:
|
||||||
|
manifest = load_manifest(manifest_path)
|
||||||
|
config = resolve_config(manifest)
|
||||||
|
store = _build_merged_snapshot_store(
|
||||||
|
manifest,
|
||||||
|
config,
|
||||||
|
snapshot_dir_override=None,
|
||||||
|
).slice_by_date_range(start_date, end_date)
|
||||||
|
return {str(symbol).upper() for symbol in store._bars.keys()}
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser(description="Build daily Form 4 same-day cluster parquet")
|
||||||
|
parser.add_argument("--config", required=True, help="Experiment manifest JSON path")
|
||||||
|
parser.add_argument("--start", required=True, help="Start date (YYYY, YYYY-MM, YYYY-MM-DD)")
|
||||||
|
parser.add_argument("--end", required=True, help="End date (YYYY, YYYY-MM, YYYY-MM-DD)")
|
||||||
|
parser.add_argument("--cache-dir", default="data/cache/form4", help="SEC zip cache dir")
|
||||||
|
parser.add_argument("--output", default="data/reference/form4_daily_events_pit.parquet", help="Output parquet path")
|
||||||
|
parser.add_argument("--user-agent", default=_DEFAULT_USER_AGENT, help="SEC User-Agent header")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
configure_logging("INFO")
|
||||||
|
start_date = _parse_date(args.start)
|
||||||
|
end_date = _parse_date(args.end, is_end=True)
|
||||||
|
allowed_symbols = _allowed_symbols_from_manifest(args.config, start_date, end_date)
|
||||||
|
transactions, skipped_quarters = _load_form4_transactions(
|
||||||
|
cache_dir=Path(args.cache_dir),
|
||||||
|
start_date=start_date,
|
||||||
|
end_date=end_date,
|
||||||
|
allowed_symbols=allowed_symbols,
|
||||||
|
user_agent=args.user_agent,
|
||||||
|
)
|
||||||
|
events = _aggregate_daily_events(transactions)
|
||||||
|
output_path = Path(args.output)
|
||||||
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
table = pa.Table.from_pylist(events)
|
||||||
|
pq.write_table(table, str(output_path))
|
||||||
|
print(
|
||||||
|
{
|
||||||
|
"output": str(output_path),
|
||||||
|
"symbols": len({row["symbol"] for row in events}),
|
||||||
|
"events": len(events),
|
||||||
|
"transactions": len(transactions),
|
||||||
|
"skipped_quarters": skipped_quarters,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@ -0,0 +1,95 @@
|
|||||||
|
"""Direct backtest subprocess runner.
|
||||||
|
|
||||||
|
Invoked by the web server as a subprocess:
|
||||||
|
python -m apps.web.direct_runner TASK_ID CONFIG_PATH CAPITAL START_DATE END_DATE RESULT_FILE [--parking PRESET] [--idle-alpha PRESET] [--form4-sleeve PRESET]
|
||||||
|
|
||||||
|
Runs run_backtest_session_sync() and saves the result JSON to RESULT_FILE.
|
||||||
|
Exits 0 on success, non-zero on failure.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import datetime as dt
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def _json_default(obj):
|
||||||
|
if isinstance(obj, (dt.date, dt.datetime)):
|
||||||
|
return obj.isoformat()
|
||||||
|
raise TypeError(f"Object of type {type(obj)} is not JSON serializable")
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
if len(sys.argv) < 7:
|
||||||
|
print(
|
||||||
|
"Usage: direct_runner TASK_ID CONFIG_PATH CAPITAL START_DATE END_DATE RESULT_FILE [--parking PRESET] [--idle-alpha PRESET] [--form4-sleeve PRESET]",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
sys.exit(2)
|
||||||
|
|
||||||
|
task_id = sys.argv[1]
|
||||||
|
config_path = sys.argv[2]
|
||||||
|
capital = float(sys.argv[3])
|
||||||
|
start_date = dt.date.fromisoformat(sys.argv[4])
|
||||||
|
end_date = dt.date.fromisoformat(sys.argv[5])
|
||||||
|
result_file = Path(sys.argv[6])
|
||||||
|
|
||||||
|
# Optional --parking PRESET, --idle-alpha PRESET, --form4-sleeve PRESET, and --snapshot-id ID
|
||||||
|
parking_preset = None
|
||||||
|
idle_alpha_preset = None
|
||||||
|
form4_sleeve_preset = None
|
||||||
|
snapshot_id_override = None
|
||||||
|
remaining = sys.argv[7:]
|
||||||
|
i = 0
|
||||||
|
while i < len(remaining):
|
||||||
|
if remaining[i] == "--parking" and i + 1 < len(remaining):
|
||||||
|
parking_preset = remaining[i + 1]
|
||||||
|
i += 2
|
||||||
|
elif remaining[i] == "--idle-alpha" and i + 1 < len(remaining):
|
||||||
|
idle_alpha_preset = remaining[i + 1]
|
||||||
|
i += 2
|
||||||
|
elif remaining[i] == "--form4-sleeve" and i + 1 < len(remaining):
|
||||||
|
form4_sleeve_preset = remaining[i + 1]
|
||||||
|
i += 2
|
||||||
|
elif remaining[i] == "--snapshot-id" and i + 1 < len(remaining):
|
||||||
|
snapshot_id_override = remaining[i + 1]
|
||||||
|
i += 2
|
||||||
|
else:
|
||||||
|
i += 1
|
||||||
|
|
||||||
|
print(f"[direct] {task_id} · {Path(config_path).stem} · {start_date}→{end_date}" +
|
||||||
|
(f" · parking={parking_preset}" if parking_preset else "") +
|
||||||
|
(f" · idle_alpha={idle_alpha_preset}" if idle_alpha_preset else "") +
|
||||||
|
(f" · form4={form4_sleeve_preset}" if form4_sleeve_preset else "") +
|
||||||
|
(f" · snapshot={snapshot_id_override}" if snapshot_id_override else ""))
|
||||||
|
sys.stdout.flush()
|
||||||
|
|
||||||
|
from apps.paper_trader.backtest_sim import run_backtest_session_sync
|
||||||
|
|
||||||
|
result = run_backtest_session_sync(
|
||||||
|
session_name=Path(config_path).stem,
|
||||||
|
config_path=config_path,
|
||||||
|
initial_equity=capital,
|
||||||
|
start_date=start_date,
|
||||||
|
end_date=end_date,
|
||||||
|
parking_preset=parking_preset,
|
||||||
|
idle_alpha_preset=idle_alpha_preset,
|
||||||
|
form4_sleeve_preset=form4_sleeve_preset,
|
||||||
|
snapshot_id_override=snapshot_id_override,
|
||||||
|
)
|
||||||
|
|
||||||
|
result_file.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
result_file.write_text(json.dumps(result, default=_json_default, indent=2))
|
||||||
|
|
||||||
|
s = result.get("summary", {})
|
||||||
|
print(
|
||||||
|
f"[direct] done · return={s.get('return_pct', 0):+.2f}% "
|
||||||
|
f"trades={s.get('trade_count', 0)} "
|
||||||
|
f"sharpe={s.get('sharpe', 0):.2f}"
|
||||||
|
)
|
||||||
|
sys.stdout.flush()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
File diff suppressed because one or more lines are too long
@ -0,0 +1,179 @@
|
|||||||
|
"""Point-in-time Form 4 cluster helpers for leakage-safe residual cash sleeves."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import datetime as dt
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from functools import lru_cache
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Iterable
|
||||||
|
|
||||||
|
import pyarrow.parquet as pq
|
||||||
|
|
||||||
|
from libs.common.logging import get_logger
|
||||||
|
|
||||||
|
logger = get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Form4ClusterEntry:
|
||||||
|
symbol: str
|
||||||
|
filing_date: dt.date
|
||||||
|
as_of_date: dt.date
|
||||||
|
total_value: float
|
||||||
|
owner_count: int
|
||||||
|
transaction_count: int
|
||||||
|
event_day_count: int
|
||||||
|
max_purchase_pct: float
|
||||||
|
median_purchase_pct: float
|
||||||
|
weighted_purchase_pct: float
|
||||||
|
max_lag_days: int | None = None
|
||||||
|
min_lag_days: int | None = None
|
||||||
|
has_officer_or_director: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
def _coerce_date(value: Any) -> dt.date | None:
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
if isinstance(value, dt.datetime):
|
||||||
|
return value.date()
|
||||||
|
if isinstance(value, dt.date):
|
||||||
|
return value
|
||||||
|
text = str(value).strip()
|
||||||
|
if not text:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return dt.date.fromisoformat(text[:10])
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _coerce_float(value: Any) -> float | None:
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return float(value)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _coerce_int(value: Any) -> int | None:
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return int(value)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class PointInTimeForm4Calendar:
|
||||||
|
"""Latest-known same-day Form 4 cluster events by filing date."""
|
||||||
|
|
||||||
|
def __init__(self, entries: Iterable[Form4ClusterEntry]) -> None:
|
||||||
|
grouped: dict[dt.date, list[Form4ClusterEntry]] = {}
|
||||||
|
for entry in entries:
|
||||||
|
grouped.setdefault(entry.filing_date, []).append(entry)
|
||||||
|
self._entries_by_filing_date = {
|
||||||
|
filing_date: tuple(sorted(
|
||||||
|
filing_entries,
|
||||||
|
key=lambda item: (
|
||||||
|
item.symbol,
|
||||||
|
-item.owner_count,
|
||||||
|
-item.total_value,
|
||||||
|
-item.weighted_purchase_pct,
|
||||||
|
),
|
||||||
|
))
|
||||||
|
for filing_date, filing_entries in grouped.items()
|
||||||
|
}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_parquet(cls, path: Path) -> PointInTimeForm4Calendar:
|
||||||
|
table = pq.read_table(str(path))
|
||||||
|
rows = table.to_pylist()
|
||||||
|
entries: list[Form4ClusterEntry] = []
|
||||||
|
for row in rows:
|
||||||
|
symbol = str(row.get("symbol") or "").strip().upper()
|
||||||
|
filing_date = _coerce_date(row.get("filing_date"))
|
||||||
|
as_of_date = _coerce_date(row.get("as_of_date") or row.get("filing_date"))
|
||||||
|
total_value = _coerce_float(row.get("total_value"))
|
||||||
|
owner_count = _coerce_int(row.get("owner_count"))
|
||||||
|
transaction_count = _coerce_int(row.get("transaction_count"))
|
||||||
|
event_day_count = _coerce_int(row.get("event_day_count"))
|
||||||
|
max_purchase_pct = _coerce_float(row.get("max_purchase_pct"))
|
||||||
|
median_purchase_pct = _coerce_float(row.get("median_purchase_pct"))
|
||||||
|
weighted_purchase_pct = _coerce_float(row.get("weighted_purchase_pct"))
|
||||||
|
if (
|
||||||
|
not symbol
|
||||||
|
or filing_date is None
|
||||||
|
or as_of_date is None
|
||||||
|
or total_value is None
|
||||||
|
or owner_count is None
|
||||||
|
or transaction_count is None
|
||||||
|
or event_day_count is None
|
||||||
|
or max_purchase_pct is None
|
||||||
|
or median_purchase_pct is None
|
||||||
|
or weighted_purchase_pct is None
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
entries.append(
|
||||||
|
Form4ClusterEntry(
|
||||||
|
symbol=symbol,
|
||||||
|
filing_date=filing_date,
|
||||||
|
as_of_date=as_of_date,
|
||||||
|
total_value=total_value,
|
||||||
|
owner_count=owner_count,
|
||||||
|
transaction_count=transaction_count,
|
||||||
|
event_day_count=event_day_count,
|
||||||
|
max_purchase_pct=max_purchase_pct,
|
||||||
|
median_purchase_pct=median_purchase_pct,
|
||||||
|
weighted_purchase_pct=weighted_purchase_pct,
|
||||||
|
max_lag_days=_coerce_int(row.get("max_lag_days")),
|
||||||
|
min_lag_days=_coerce_int(row.get("min_lag_days")),
|
||||||
|
has_officer_or_director=bool(row.get("has_officer_or_director") or False),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
logger.info(
|
||||||
|
"pit_form4_calendar_loaded",
|
||||||
|
path=str(path),
|
||||||
|
rows=len(entries),
|
||||||
|
filing_dates=len({entry.filing_date for entry in entries}),
|
||||||
|
symbols=len({entry.symbol for entry in entries}),
|
||||||
|
)
|
||||||
|
return cls(entries)
|
||||||
|
|
||||||
|
def get_events_between(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
start_filing_date: dt.date,
|
||||||
|
end_filing_date: dt.date,
|
||||||
|
symbols: Iterable[str] | None = None,
|
||||||
|
) -> list[Form4ClusterEntry]:
|
||||||
|
symbol_filter = {
|
||||||
|
str(symbol).strip().upper()
|
||||||
|
for symbol in (symbols or [])
|
||||||
|
if str(symbol).strip()
|
||||||
|
}
|
||||||
|
rows: list[Form4ClusterEntry] = []
|
||||||
|
current = start_filing_date
|
||||||
|
while current <= end_filing_date:
|
||||||
|
for entry in self._entries_by_filing_date.get(current, ()):
|
||||||
|
if symbol_filter and entry.symbol not in symbol_filter:
|
||||||
|
continue
|
||||||
|
rows.append(entry)
|
||||||
|
current += dt.timedelta(days=1)
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
@lru_cache(maxsize=8)
|
||||||
|
def load_pit_form4_calendar(path_str: str) -> PointInTimeForm4Calendar | None:
|
||||||
|
path = Path(path_str)
|
||||||
|
if not path.exists():
|
||||||
|
logger.info("pit_form4_calendar_missing", path=str(path))
|
||||||
|
return None
|
||||||
|
return PointInTimeForm4Calendar.from_parquet(path)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"Form4ClusterEntry",
|
||||||
|
"PointInTimeForm4Calendar",
|
||||||
|
"load_pit_form4_calendar",
|
||||||
|
]
|
||||||
@ -0,0 +1,94 @@
|
|||||||
|
"""Dividend calendar Oracle service methods."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import datetime as dt
|
||||||
|
from typing import Sequence
|
||||||
|
|
||||||
|
from libs.oracle_client.client import OracleClient
|
||||||
|
from libs.oracle_client.models import (
|
||||||
|
DividendCalendarEntry,
|
||||||
|
DividendHistoryResponse,
|
||||||
|
DividendIngestResponse,
|
||||||
|
DividendUpcomingResponse,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _date_param(value: dt.date | str | None) -> str | None:
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
if isinstance(value, dt.date):
|
||||||
|
return value.isoformat()
|
||||||
|
return str(value)
|
||||||
|
|
||||||
|
|
||||||
|
class DividendService:
|
||||||
|
def __init__(self, client: OracleClient) -> None:
|
||||||
|
self._client = client
|
||||||
|
|
||||||
|
async def get_upcoming(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
as_of_date: dt.date | str | None = None,
|
||||||
|
from_ex_date: dt.date | str | None = None,
|
||||||
|
to_ex_date: dt.date | str | None = None,
|
||||||
|
symbols: Sequence[str] | None = None,
|
||||||
|
limit: int = 500,
|
||||||
|
force_refresh: bool = False,
|
||||||
|
) -> DividendUpcomingResponse:
|
||||||
|
params: dict[str, object] = {
|
||||||
|
"limit": int(limit),
|
||||||
|
"force_refresh": bool(force_refresh),
|
||||||
|
}
|
||||||
|
as_of = _date_param(as_of_date)
|
||||||
|
from_ex = _date_param(from_ex_date)
|
||||||
|
to_ex = _date_param(to_ex_date)
|
||||||
|
if as_of:
|
||||||
|
params["as_of_date"] = as_of
|
||||||
|
if from_ex:
|
||||||
|
params["from_ex_date"] = from_ex
|
||||||
|
if to_ex:
|
||||||
|
params["to_ex_date"] = to_ex
|
||||||
|
if symbols:
|
||||||
|
params["symbols"] = [str(symbol).strip().upper() for symbol in symbols if str(symbol).strip()]
|
||||||
|
data = await self._client.get("/api/v1/dividends/upcoming", params=params)
|
||||||
|
entries = [DividendCalendarEntry.model_validate(item) for item in data.get("dividends", [])]
|
||||||
|
return DividendUpcomingResponse(
|
||||||
|
dividends=entries,
|
||||||
|
total_count=int(data.get("total_count", len(entries))),
|
||||||
|
metadata=dict(data.get("metadata") or {}),
|
||||||
|
)
|
||||||
|
|
||||||
|
async def get_history(
|
||||||
|
self,
|
||||||
|
symbol: str,
|
||||||
|
*,
|
||||||
|
limit: int = 1000,
|
||||||
|
force_refresh: bool = False,
|
||||||
|
) -> DividendHistoryResponse:
|
||||||
|
data = await self._client.get(
|
||||||
|
f"/api/v1/dividends/history/{symbol}",
|
||||||
|
params={"limit": int(limit), "force_refresh": bool(force_refresh)},
|
||||||
|
)
|
||||||
|
entries = [DividendCalendarEntry.model_validate(item) for item in data.get("dividends", [])]
|
||||||
|
annual_yield_estimate = data.get("annual_yield_estimate")
|
||||||
|
return DividendHistoryResponse(
|
||||||
|
symbol=data.get("symbol", symbol),
|
||||||
|
dividends=entries,
|
||||||
|
total_count=int(data.get("total_count", len(entries))),
|
||||||
|
annual_yield_estimate=float(annual_yield_estimate) if annual_yield_estimate is not None else None,
|
||||||
|
metadata=dict(data.get("metadata") or {}),
|
||||||
|
)
|
||||||
|
|
||||||
|
async def ingest(
|
||||||
|
self,
|
||||||
|
symbols: Sequence[str],
|
||||||
|
*,
|
||||||
|
force_refresh: bool = False,
|
||||||
|
) -> DividendIngestResponse:
|
||||||
|
payload = {
|
||||||
|
"symbols": [str(symbol).strip().upper() for symbol in symbols if str(symbol).strip()],
|
||||||
|
"force_refresh": bool(force_refresh),
|
||||||
|
}
|
||||||
|
data = await self._client.post("/api/v1/dividends/admin/ingest", json=payload)
|
||||||
|
return DividendIngestResponse.model_validate(data)
|
||||||
Loading…
Reference in New Issue