Add Form4 residual-cash sleeve and UI support
I Luk Kim
4 months ago
parent
86d55e01f9
commit
72681e69e5
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,299 @@
|
||||
"""Build leakage-safe daily Form 4 cluster parquet from SEC flat files.
|
||||
|
||||
The output is a daily same-day cluster cache keyed by filing date. Runtime code
|
||||
uses this cache conservatively on the next trading day only.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import datetime as dt
|
||||
import io
|
||||
import math
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pyarrow as pa
|
||||
import pyarrow.parquet as pq
|
||||
|
||||
from apps.backtester.run import _build_merged_snapshot_store, load_manifest, resolve_config
|
||||
from libs.common.logging import configure_logging
|
||||
|
||||
_DEFAULT_USER_AGENT = "fithia2-form4-cache/1.0 (local research; contact: dev@example.com)"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RawForm4Transaction:
|
||||
symbol: str
|
||||
filing_date: dt.date
|
||||
transaction_date: dt.date
|
||||
owner_cik: str
|
||||
owner_relationship: str
|
||||
owner_title: str
|
||||
shares: float
|
||||
price: float
|
||||
total_value: float
|
||||
shares_owned_following: float
|
||||
purchase_pct_of_holding: float
|
||||
|
||||
|
||||
def _parse_date(value: str, *, is_end: bool = False) -> dt.date:
|
||||
parts = value.split("-")
|
||||
if len(parts) == 1 and len(value) == 4 and value.isdigit():
|
||||
year = int(value)
|
||||
return dt.date(year, 12, 31) if is_end else dt.date(year, 1, 1)
|
||||
if len(parts) == 2 and all(part.isdigit() for part in parts):
|
||||
year = int(parts[0])
|
||||
month = int(parts[1])
|
||||
if is_end:
|
||||
next_month = dt.date(year + (month // 12), (month % 12) + 1, 1)
|
||||
return next_month - dt.timedelta(days=1)
|
||||
return dt.date(year, month, 1)
|
||||
return dt.date.fromisoformat(value)
|
||||
|
||||
|
||||
def _quarter_range(start_date: dt.date, end_date: dt.date) -> list[tuple[int, int]]:
|
||||
year = start_date.year
|
||||
quarter = (start_date.month - 1) // 3 + 1
|
||||
end_key = (end_date.year, (end_date.month - 1) // 3 + 1)
|
||||
quarters: list[tuple[int, int]] = []
|
||||
while (year, quarter) <= end_key:
|
||||
quarters.append((year, quarter))
|
||||
quarter += 1
|
||||
if quarter == 5:
|
||||
year += 1
|
||||
quarter = 1
|
||||
return quarters
|
||||
|
||||
|
||||
def _quarter_zip_path(cache_dir: Path, year: int, quarter: int) -> Path:
|
||||
return cache_dir / f"{year}q{quarter}_form345.zip"
|
||||
|
||||
|
||||
def _quarter_zip_url(year: int, quarter: int) -> str:
|
||||
return (
|
||||
"https://www.sec.gov/files/structureddata/data/"
|
||||
f"insider-transactions-data-sets/{year}q{quarter}_form345.zip"
|
||||
)
|
||||
|
||||
|
||||
def _ensure_quarter_zip(cache_dir: Path, year: int, quarter: int, *, user_agent: str) -> Path | None:
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = _quarter_zip_path(cache_dir, year, quarter)
|
||||
if path.exists() and path.stat().st_size > 0:
|
||||
return path
|
||||
request = urllib.request.Request(_quarter_zip_url(year, quarter), headers={"User-Agent": user_agent})
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=90) as response:
|
||||
data = response.read()
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code == 404:
|
||||
return None
|
||||
raise
|
||||
path.write_bytes(data)
|
||||
return path
|
||||
|
||||
|
||||
def _parse_sec_date(value: str | None) -> dt.date | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return dt.datetime.strptime(value, "%d-%b-%Y").date()
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _coerce_float(value: Any) -> float | None:
|
||||
try:
|
||||
result = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if math.isnan(result) or math.isinf(result):
|
||||
return None
|
||||
return result
|
||||
|
||||
|
||||
def _is_officer_or_director(relationship: str, title: str) -> bool:
|
||||
combined = f"{relationship} {title}".strip().lower()
|
||||
return any(token in combined for token in ("director", "officer", "chief", "ceo", "cfo", "coo", "president", "chair"))
|
||||
|
||||
|
||||
def _load_form4_transactions(
|
||||
*,
|
||||
cache_dir: Path,
|
||||
start_date: dt.date,
|
||||
end_date: dt.date,
|
||||
allowed_symbols: set[str],
|
||||
user_agent: str,
|
||||
) -> tuple[list[RawForm4Transaction], list[str]]:
|
||||
transactions: list[RawForm4Transaction] = []
|
||||
skipped_quarters: list[str] = []
|
||||
|
||||
for year, quarter in _quarter_range(start_date, end_date):
|
||||
path = _ensure_quarter_zip(cache_dir, year, quarter, user_agent=user_agent)
|
||||
if path is None:
|
||||
skipped_quarters.append(f"{year}Q{quarter}")
|
||||
continue
|
||||
with zipfile.ZipFile(path) as zf:
|
||||
submissions: dict[str, dict[str, Any]] = {}
|
||||
with zf.open("SUBMISSION.tsv") as handle:
|
||||
reader = csv.DictReader(io.TextIOWrapper(handle, encoding="utf-8", newline=""), delimiter="\t")
|
||||
for row in reader:
|
||||
symbol = str(row.get("ISSUERTRADINGSYMBOL") or "").strip().upper()
|
||||
if not symbol or symbol not in allowed_symbols:
|
||||
continue
|
||||
if str(row.get("DOCUMENT_TYPE") or "").strip().upper() != "4":
|
||||
continue
|
||||
filing_date = _parse_sec_date(row.get("FILING_DATE"))
|
||||
if filing_date is None or filing_date < start_date or filing_date > end_date:
|
||||
continue
|
||||
submissions[str(row["ACCESSION_NUMBER"])] = {
|
||||
"symbol": symbol,
|
||||
"filing_date": filing_date,
|
||||
}
|
||||
|
||||
if not submissions:
|
||||
continue
|
||||
|
||||
relationships: dict[str, list[tuple[str, str, str]]] = defaultdict(list)
|
||||
with zf.open("REPORTINGOWNER.tsv") as handle:
|
||||
reader = csv.DictReader(io.TextIOWrapper(handle, encoding="utf-8", newline=""), delimiter="\t")
|
||||
for row in reader:
|
||||
accession = str(row["ACCESSION_NUMBER"])
|
||||
if accession not in submissions:
|
||||
continue
|
||||
owner_cik = str(row.get("RPTOWNERCIK") or "").strip()
|
||||
relationship = str(row.get("OFFICER_TITLE") or "").strip()
|
||||
title = str(row.get("OTHER_TEXT") or "").strip()
|
||||
relationships[accession].append((owner_cik, relationship, title))
|
||||
|
||||
with zf.open("NONDERIV_TRANS.tsv") as handle:
|
||||
reader = csv.DictReader(io.TextIOWrapper(handle, encoding="utf-8", newline=""), delimiter="\t")
|
||||
for row in reader:
|
||||
accession = str(row["ACCESSION_NUMBER"])
|
||||
submission = submissions.get(accession)
|
||||
if submission is None:
|
||||
continue
|
||||
transaction_code = str(row.get("TRANS_CODE") or "").strip().upper()
|
||||
if transaction_code != "P":
|
||||
continue
|
||||
shares = _coerce_float(row.get("TRANS_SHARES"))
|
||||
price = _coerce_float(row.get("TRANS_PRICEPERSHARE"))
|
||||
if shares is None or price is None or shares <= 0 or price <= 0:
|
||||
continue
|
||||
transaction_date = _parse_sec_date(row.get("TRANS_DATE")) or submission["filing_date"]
|
||||
shares_owned_following = _coerce_float(row.get("SHRS_OWND_FOLWNG_TRANS")) or 0.0
|
||||
purchase_pct = 0.0
|
||||
if shares_owned_following > 0:
|
||||
purchase_pct = min(1.0, shares / shares_owned_following)
|
||||
rels = relationships.get(accession) or [("", "", "")]
|
||||
for owner_cik, relationship, title in rels:
|
||||
transactions.append(
|
||||
RawForm4Transaction(
|
||||
symbol=submission["symbol"],
|
||||
filing_date=submission["filing_date"],
|
||||
transaction_date=transaction_date,
|
||||
owner_cik=owner_cik,
|
||||
owner_relationship=relationship,
|
||||
owner_title=title,
|
||||
shares=shares,
|
||||
price=price,
|
||||
total_value=shares * price,
|
||||
shares_owned_following=shares_owned_following,
|
||||
purchase_pct_of_holding=purchase_pct,
|
||||
)
|
||||
)
|
||||
|
||||
return transactions, skipped_quarters
|
||||
|
||||
|
||||
def _aggregate_daily_events(transactions: list[RawForm4Transaction]) -> list[dict[str, Any]]:
|
||||
grouped: dict[tuple[dt.date, str], list[RawForm4Transaction]] = defaultdict(list)
|
||||
for row in transactions:
|
||||
grouped[(row.filing_date, row.symbol)].append(row)
|
||||
|
||||
events: list[dict[str, Any]] = []
|
||||
for (filing_date, symbol), rows in grouped.items():
|
||||
owner_keys = {row.owner_cik or f"{symbol}:{idx}" for idx, row in enumerate(rows)}
|
||||
purchase_values = [row.purchase_pct_of_holding for row in rows if row.purchase_pct_of_holding > 0]
|
||||
lag_values = [(row.filing_date - row.transaction_date).days for row in rows]
|
||||
events.append(
|
||||
{
|
||||
"symbol": symbol,
|
||||
"filing_date": filing_date.isoformat(),
|
||||
"as_of_date": filing_date.isoformat(),
|
||||
"total_value": round(sum(row.total_value for row in rows), 2),
|
||||
"owner_count": len(owner_keys),
|
||||
"transaction_count": len(rows),
|
||||
"event_day_count": 1,
|
||||
"max_purchase_pct": max(purchase_values) if purchase_values else 0.0,
|
||||
"median_purchase_pct": sorted(purchase_values)[len(purchase_values) // 2] if purchase_values else 0.0,
|
||||
"weighted_purchase_pct": max(purchase_values) if purchase_values else 0.0,
|
||||
"max_lag_days": max(lag_values) if lag_values else None,
|
||||
"min_lag_days": min(lag_values) if lag_values else None,
|
||||
"has_officer_or_director": any(
|
||||
_is_officer_or_director(row.owner_relationship, row.owner_title)
|
||||
for row in rows
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
events.sort(key=lambda row: (row["filing_date"], row["symbol"]))
|
||||
return events
|
||||
|
||||
|
||||
def _allowed_symbols_from_manifest(manifest_path: str, start_date: dt.date, end_date: dt.date) -> set[str]:
|
||||
manifest = load_manifest(manifest_path)
|
||||
config = resolve_config(manifest)
|
||||
store = _build_merged_snapshot_store(
|
||||
manifest,
|
||||
config,
|
||||
snapshot_dir_override=None,
|
||||
).slice_by_date_range(start_date, end_date)
|
||||
return {str(symbol).upper() for symbol in store._bars.keys()}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Build daily Form 4 same-day cluster parquet")
|
||||
parser.add_argument("--config", required=True, help="Experiment manifest JSON path")
|
||||
parser.add_argument("--start", required=True, help="Start date (YYYY, YYYY-MM, YYYY-MM-DD)")
|
||||
parser.add_argument("--end", required=True, help="End date (YYYY, YYYY-MM, YYYY-MM-DD)")
|
||||
parser.add_argument("--cache-dir", default="data/cache/form4", help="SEC zip cache dir")
|
||||
parser.add_argument("--output", default="data/reference/form4_daily_events_pit.parquet", help="Output parquet path")
|
||||
parser.add_argument("--user-agent", default=_DEFAULT_USER_AGENT, help="SEC User-Agent header")
|
||||
args = parser.parse_args()
|
||||
|
||||
configure_logging("INFO")
|
||||
start_date = _parse_date(args.start)
|
||||
end_date = _parse_date(args.end, is_end=True)
|
||||
allowed_symbols = _allowed_symbols_from_manifest(args.config, start_date, end_date)
|
||||
transactions, skipped_quarters = _load_form4_transactions(
|
||||
cache_dir=Path(args.cache_dir),
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
allowed_symbols=allowed_symbols,
|
||||
user_agent=args.user_agent,
|
||||
)
|
||||
events = _aggregate_daily_events(transactions)
|
||||
output_path = Path(args.output)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
table = pa.Table.from_pylist(events)
|
||||
pq.write_table(table, str(output_path))
|
||||
print(
|
||||
{
|
||||
"output": str(output_path),
|
||||
"symbols": len({row["symbol"] for row in events}),
|
||||
"events": len(events),
|
||||
"transactions": len(transactions),
|
||||
"skipped_quarters": skipped_quarters,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,95 @@
|
||||
"""Direct backtest subprocess runner.
|
||||
|
||||
Invoked by the web server as a subprocess:
|
||||
python -m apps.web.direct_runner TASK_ID CONFIG_PATH CAPITAL START_DATE END_DATE RESULT_FILE [--parking PRESET] [--idle-alpha PRESET] [--form4-sleeve PRESET]
|
||||
|
||||
Runs run_backtest_session_sync() and saves the result JSON to RESULT_FILE.
|
||||
Exits 0 on success, non-zero on failure.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _json_default(obj):
|
||||
if isinstance(obj, (dt.date, dt.datetime)):
|
||||
return obj.isoformat()
|
||||
raise TypeError(f"Object of type {type(obj)} is not JSON serializable")
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 7:
|
||||
print(
|
||||
"Usage: direct_runner TASK_ID CONFIG_PATH CAPITAL START_DATE END_DATE RESULT_FILE [--parking PRESET] [--idle-alpha PRESET] [--form4-sleeve PRESET]",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(2)
|
||||
|
||||
task_id = sys.argv[1]
|
||||
config_path = sys.argv[2]
|
||||
capital = float(sys.argv[3])
|
||||
start_date = dt.date.fromisoformat(sys.argv[4])
|
||||
end_date = dt.date.fromisoformat(sys.argv[5])
|
||||
result_file = Path(sys.argv[6])
|
||||
|
||||
# Optional --parking PRESET, --idle-alpha PRESET, --form4-sleeve PRESET, and --snapshot-id ID
|
||||
parking_preset = None
|
||||
idle_alpha_preset = None
|
||||
form4_sleeve_preset = None
|
||||
snapshot_id_override = None
|
||||
remaining = sys.argv[7:]
|
||||
i = 0
|
||||
while i < len(remaining):
|
||||
if remaining[i] == "--parking" and i + 1 < len(remaining):
|
||||
parking_preset = remaining[i + 1]
|
||||
i += 2
|
||||
elif remaining[i] == "--idle-alpha" and i + 1 < len(remaining):
|
||||
idle_alpha_preset = remaining[i + 1]
|
||||
i += 2
|
||||
elif remaining[i] == "--form4-sleeve" and i + 1 < len(remaining):
|
||||
form4_sleeve_preset = remaining[i + 1]
|
||||
i += 2
|
||||
elif remaining[i] == "--snapshot-id" and i + 1 < len(remaining):
|
||||
snapshot_id_override = remaining[i + 1]
|
||||
i += 2
|
||||
else:
|
||||
i += 1
|
||||
|
||||
print(f"[direct] {task_id} · {Path(config_path).stem} · {start_date}→{end_date}" +
|
||||
(f" · parking={parking_preset}" if parking_preset else "") +
|
||||
(f" · idle_alpha={idle_alpha_preset}" if idle_alpha_preset else "") +
|
||||
(f" · form4={form4_sleeve_preset}" if form4_sleeve_preset else "") +
|
||||
(f" · snapshot={snapshot_id_override}" if snapshot_id_override else ""))
|
||||
sys.stdout.flush()
|
||||
|
||||
from apps.paper_trader.backtest_sim import run_backtest_session_sync
|
||||
|
||||
result = run_backtest_session_sync(
|
||||
session_name=Path(config_path).stem,
|
||||
config_path=config_path,
|
||||
initial_equity=capital,
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
parking_preset=parking_preset,
|
||||
idle_alpha_preset=idle_alpha_preset,
|
||||
form4_sleeve_preset=form4_sleeve_preset,
|
||||
snapshot_id_override=snapshot_id_override,
|
||||
)
|
||||
|
||||
result_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
result_file.write_text(json.dumps(result, default=_json_default, indent=2))
|
||||
|
||||
s = result.get("summary", {})
|
||||
print(
|
||||
f"[direct] done · return={s.get('return_pct', 0):+.2f}% "
|
||||
f"trades={s.get('trade_count', 0)} "
|
||||
f"sharpe={s.get('sharpe', 0):.2f}"
|
||||
)
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because one or more lines are too long
@ -0,0 +1,179 @@
|
||||
"""Point-in-time Form 4 cluster helpers for leakage-safe residual cash sleeves."""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
import pyarrow.parquet as pq
|
||||
|
||||
from libs.common.logging import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Form4ClusterEntry:
|
||||
symbol: str
|
||||
filing_date: dt.date
|
||||
as_of_date: dt.date
|
||||
total_value: float
|
||||
owner_count: int
|
||||
transaction_count: int
|
||||
event_day_count: int
|
||||
max_purchase_pct: float
|
||||
median_purchase_pct: float
|
||||
weighted_purchase_pct: float
|
||||
max_lag_days: int | None = None
|
||||
min_lag_days: int | None = None
|
||||
has_officer_or_director: bool = False
|
||||
|
||||
|
||||
def _coerce_date(value: Any) -> dt.date | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, dt.datetime):
|
||||
return value.date()
|
||||
if isinstance(value, dt.date):
|
||||
return value
|
||||
text = str(value).strip()
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
return dt.date.fromisoformat(text[:10])
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _coerce_float(value: Any) -> float | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _coerce_int(value: Any) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
class PointInTimeForm4Calendar:
|
||||
"""Latest-known same-day Form 4 cluster events by filing date."""
|
||||
|
||||
def __init__(self, entries: Iterable[Form4ClusterEntry]) -> None:
|
||||
grouped: dict[dt.date, list[Form4ClusterEntry]] = {}
|
||||
for entry in entries:
|
||||
grouped.setdefault(entry.filing_date, []).append(entry)
|
||||
self._entries_by_filing_date = {
|
||||
filing_date: tuple(sorted(
|
||||
filing_entries,
|
||||
key=lambda item: (
|
||||
item.symbol,
|
||||
-item.owner_count,
|
||||
-item.total_value,
|
||||
-item.weighted_purchase_pct,
|
||||
),
|
||||
))
|
||||
for filing_date, filing_entries in grouped.items()
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_parquet(cls, path: Path) -> PointInTimeForm4Calendar:
|
||||
table = pq.read_table(str(path))
|
||||
rows = table.to_pylist()
|
||||
entries: list[Form4ClusterEntry] = []
|
||||
for row in rows:
|
||||
symbol = str(row.get("symbol") or "").strip().upper()
|
||||
filing_date = _coerce_date(row.get("filing_date"))
|
||||
as_of_date = _coerce_date(row.get("as_of_date") or row.get("filing_date"))
|
||||
total_value = _coerce_float(row.get("total_value"))
|
||||
owner_count = _coerce_int(row.get("owner_count"))
|
||||
transaction_count = _coerce_int(row.get("transaction_count"))
|
||||
event_day_count = _coerce_int(row.get("event_day_count"))
|
||||
max_purchase_pct = _coerce_float(row.get("max_purchase_pct"))
|
||||
median_purchase_pct = _coerce_float(row.get("median_purchase_pct"))
|
||||
weighted_purchase_pct = _coerce_float(row.get("weighted_purchase_pct"))
|
||||
if (
|
||||
not symbol
|
||||
or filing_date is None
|
||||
or as_of_date is None
|
||||
or total_value is None
|
||||
or owner_count is None
|
||||
or transaction_count is None
|
||||
or event_day_count is None
|
||||
or max_purchase_pct is None
|
||||
or median_purchase_pct is None
|
||||
or weighted_purchase_pct is None
|
||||
):
|
||||
continue
|
||||
entries.append(
|
||||
Form4ClusterEntry(
|
||||
symbol=symbol,
|
||||
filing_date=filing_date,
|
||||
as_of_date=as_of_date,
|
||||
total_value=total_value,
|
||||
owner_count=owner_count,
|
||||
transaction_count=transaction_count,
|
||||
event_day_count=event_day_count,
|
||||
max_purchase_pct=max_purchase_pct,
|
||||
median_purchase_pct=median_purchase_pct,
|
||||
weighted_purchase_pct=weighted_purchase_pct,
|
||||
max_lag_days=_coerce_int(row.get("max_lag_days")),
|
||||
min_lag_days=_coerce_int(row.get("min_lag_days")),
|
||||
has_officer_or_director=bool(row.get("has_officer_or_director") or False),
|
||||
)
|
||||
)
|
||||
logger.info(
|
||||
"pit_form4_calendar_loaded",
|
||||
path=str(path),
|
||||
rows=len(entries),
|
||||
filing_dates=len({entry.filing_date for entry in entries}),
|
||||
symbols=len({entry.symbol for entry in entries}),
|
||||
)
|
||||
return cls(entries)
|
||||
|
||||
def get_events_between(
|
||||
self,
|
||||
*,
|
||||
start_filing_date: dt.date,
|
||||
end_filing_date: dt.date,
|
||||
symbols: Iterable[str] | None = None,
|
||||
) -> list[Form4ClusterEntry]:
|
||||
symbol_filter = {
|
||||
str(symbol).strip().upper()
|
||||
for symbol in (symbols or [])
|
||||
if str(symbol).strip()
|
||||
}
|
||||
rows: list[Form4ClusterEntry] = []
|
||||
current = start_filing_date
|
||||
while current <= end_filing_date:
|
||||
for entry in self._entries_by_filing_date.get(current, ()):
|
||||
if symbol_filter and entry.symbol not in symbol_filter:
|
||||
continue
|
||||
rows.append(entry)
|
||||
current += dt.timedelta(days=1)
|
||||
return rows
|
||||
|
||||
|
||||
@lru_cache(maxsize=8)
|
||||
def load_pit_form4_calendar(path_str: str) -> PointInTimeForm4Calendar | None:
|
||||
path = Path(path_str)
|
||||
if not path.exists():
|
||||
logger.info("pit_form4_calendar_missing", path=str(path))
|
||||
return None
|
||||
return PointInTimeForm4Calendar.from_parquet(path)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Form4ClusterEntry",
|
||||
"PointInTimeForm4Calendar",
|
||||
"load_pit_form4_calendar",
|
||||
]
|
||||
@ -0,0 +1,94 @@
|
||||
"""Dividend calendar Oracle service methods."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
from typing import Sequence
|
||||
|
||||
from libs.oracle_client.client import OracleClient
|
||||
from libs.oracle_client.models import (
|
||||
DividendCalendarEntry,
|
||||
DividendHistoryResponse,
|
||||
DividendIngestResponse,
|
||||
DividendUpcomingResponse,
|
||||
)
|
||||
|
||||
|
||||
def _date_param(value: dt.date | str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, dt.date):
|
||||
return value.isoformat()
|
||||
return str(value)
|
||||
|
||||
|
||||
class DividendService:
|
||||
def __init__(self, client: OracleClient) -> None:
|
||||
self._client = client
|
||||
|
||||
async def get_upcoming(
|
||||
self,
|
||||
*,
|
||||
as_of_date: dt.date | str | None = None,
|
||||
from_ex_date: dt.date | str | None = None,
|
||||
to_ex_date: dt.date | str | None = None,
|
||||
symbols: Sequence[str] | None = None,
|
||||
limit: int = 500,
|
||||
force_refresh: bool = False,
|
||||
) -> DividendUpcomingResponse:
|
||||
params: dict[str, object] = {
|
||||
"limit": int(limit),
|
||||
"force_refresh": bool(force_refresh),
|
||||
}
|
||||
as_of = _date_param(as_of_date)
|
||||
from_ex = _date_param(from_ex_date)
|
||||
to_ex = _date_param(to_ex_date)
|
||||
if as_of:
|
||||
params["as_of_date"] = as_of
|
||||
if from_ex:
|
||||
params["from_ex_date"] = from_ex
|
||||
if to_ex:
|
||||
params["to_ex_date"] = to_ex
|
||||
if symbols:
|
||||
params["symbols"] = [str(symbol).strip().upper() for symbol in symbols if str(symbol).strip()]
|
||||
data = await self._client.get("/api/v1/dividends/upcoming", params=params)
|
||||
entries = [DividendCalendarEntry.model_validate(item) for item in data.get("dividends", [])]
|
||||
return DividendUpcomingResponse(
|
||||
dividends=entries,
|
||||
total_count=int(data.get("total_count", len(entries))),
|
||||
metadata=dict(data.get("metadata") or {}),
|
||||
)
|
||||
|
||||
async def get_history(
|
||||
self,
|
||||
symbol: str,
|
||||
*,
|
||||
limit: int = 1000,
|
||||
force_refresh: bool = False,
|
||||
) -> DividendHistoryResponse:
|
||||
data = await self._client.get(
|
||||
f"/api/v1/dividends/history/{symbol}",
|
||||
params={"limit": int(limit), "force_refresh": bool(force_refresh)},
|
||||
)
|
||||
entries = [DividendCalendarEntry.model_validate(item) for item in data.get("dividends", [])]
|
||||
annual_yield_estimate = data.get("annual_yield_estimate")
|
||||
return DividendHistoryResponse(
|
||||
symbol=data.get("symbol", symbol),
|
||||
dividends=entries,
|
||||
total_count=int(data.get("total_count", len(entries))),
|
||||
annual_yield_estimate=float(annual_yield_estimate) if annual_yield_estimate is not None else None,
|
||||
metadata=dict(data.get("metadata") or {}),
|
||||
)
|
||||
|
||||
async def ingest(
|
||||
self,
|
||||
symbols: Sequence[str],
|
||||
*,
|
||||
force_refresh: bool = False,
|
||||
) -> DividendIngestResponse:
|
||||
payload = {
|
||||
"symbols": [str(symbol).strip().upper() for symbol in symbols if str(symbol).strip()],
|
||||
"force_refresh": bool(force_refresh),
|
||||
}
|
||||
data = await self._client.post("/api/v1/dividends/admin/ingest", json=payload)
|
||||
return DividendIngestResponse.model_validate(data)
|
||||
Loading…
Reference in New Issue