You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
112 lines
3.9 KiB
Python
112 lines
3.9 KiB
Python
#!/usr/bin/env python
|
|
"""Normalize provider exports into the leakage-safe PIT earnings calendar schema."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import datetime as dt
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pyarrow as pa
|
|
import pyarrow.csv as pacsv
|
|
import pyarrow.parquet as pq
|
|
|
|
from libs.labeler.reaction_date import compute_reaction_date
|
|
|
|
|
|
def _read_table(path: Path) -> pa.Table:
|
|
suffix = path.suffix.lower()
|
|
if suffix == ".parquet":
|
|
return pq.read_table(str(path))
|
|
if suffix in {".csv", ".txt"}:
|
|
return pacsv.read_csv(str(path))
|
|
raise ValueError(f"Unsupported input format: {path}")
|
|
|
|
|
|
def _coerce_date(value: Any) -> str | None:
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, dt.datetime):
|
|
return value.date().isoformat()
|
|
if isinstance(value, dt.date):
|
|
return value.isoformat()
|
|
text = str(value).strip()
|
|
if not text:
|
|
return None
|
|
try:
|
|
return dt.date.fromisoformat(text[:10]).isoformat()
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def _coerce_bool(value: Any) -> bool:
|
|
if isinstance(value, bool):
|
|
return value
|
|
if value is None:
|
|
return False
|
|
text = str(value).strip().lower()
|
|
return text in {"1", "true", "t", "yes", "y"}
|
|
|
|
|
|
def _pick(row: dict[str, Any], *names: str) -> Any:
|
|
for name in names:
|
|
if name in row and row[name] is not None:
|
|
return row[name]
|
|
return None
|
|
|
|
|
|
def build_rows(table: pa.Table) -> list[dict[str, Any]]:
|
|
rows: list[dict[str, Any]] = []
|
|
for row in table.to_pylist():
|
|
symbol = str(_pick(row, "symbol", "ticker", "trade_symbol") or "").strip().upper()
|
|
as_of_date = _coerce_date(_pick(row, "as_of_date", "known_as_of_date", "published_date", "snapshot_date"))
|
|
expected_event_date = _coerce_date(_pick(row, "expected_event_date", "event_date", "earnings_date", "next_earnings_date"))
|
|
filing_time_bucket = str(_pick(row, "expected_filing_time_bucket", "filing_time_bucket", "timing_class") or "post_market")
|
|
expected_reaction_date = _coerce_date(_pick(row, "expected_reaction_date", "reaction_date"))
|
|
if expected_reaction_date is None and expected_event_date is not None:
|
|
expected_reaction_date = compute_reaction_date(
|
|
dt.date.fromisoformat(expected_event_date),
|
|
filing_time_bucket,
|
|
).isoformat()
|
|
if not symbol or as_of_date is None or expected_reaction_date is None:
|
|
continue
|
|
rows.append(
|
|
{
|
|
"symbol": symbol,
|
|
"as_of_date": as_of_date,
|
|
"expected_reaction_date": expected_reaction_date,
|
|
"expected_event_date": expected_event_date,
|
|
"filing_time_bucket": filing_time_bucket,
|
|
"confidence": _pick(row, "confidence", "resolver_confidence"),
|
|
"revision_count": _pick(row, "revision_count"),
|
|
"is_cancelled": _coerce_bool(_pick(row, "is_cancelled", "cancelled")),
|
|
"source": _pick(row, "source", "provider"),
|
|
}
|
|
)
|
|
rows.sort(key=lambda r: (r["symbol"], r["as_of_date"], r["expected_reaction_date"]))
|
|
return rows
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--input", required=True, help="Provider export CSV/Parquet")
|
|
parser.add_argument(
|
|
"--output",
|
|
default="data/reference/earnings_calendar_pit.parquet",
|
|
help="Normalized PIT calendar parquet path",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
input_path = Path(args.input)
|
|
output_path = Path(args.output)
|
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
table = _read_table(input_path)
|
|
rows = build_rows(table)
|
|
pq.write_table(pa.Table.from_pylist(rows), output_path)
|
|
print(f"wrote {len(rows)} rows to {output_path}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|