You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
555 lines
19 KiB
Python
555 lines
19 KiB
Python
"""Same-day catalyst helpers for intraday ORB research.
|
|
|
|
This module intentionally starts with the narrowest reliable catalyst source we
|
|
have today: Oracle's parsed filing events. The goal is to stop treating
|
|
premarket attention as if it were a true catalyst signal.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import datetime as dt
|
|
import gzip
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
import uuid
|
|
|
|
from libs.oracle_client import AttentionService, FilingsService, OracleClient
|
|
|
|
|
|
class FilingEventCache:
|
|
"""Disk cache for same-day filing events keyed by ticker.
|
|
|
|
Layout:
|
|
{cache_dir}/{TICKER}.json.gz
|
|
|
|
Each file stores a coverage window plus the raw event payloads returned by
|
|
Oracle. Reads support exact or subset date ranges when coverage is present.
|
|
"""
|
|
|
|
def __init__(self, cache_dir: str = "data/cache/orb_catalyst") -> None:
|
|
self._root = Path(cache_dir)
|
|
|
|
def _path(self, ticker: str) -> Path:
|
|
return self._root / f"{ticker.upper()}.json.gz"
|
|
|
|
def get(
|
|
self,
|
|
ticker: str,
|
|
start_date: str,
|
|
end_date: str,
|
|
) -> list[dict[str, Any]] | None:
|
|
p = self._path(ticker)
|
|
if not p.exists():
|
|
return None
|
|
try:
|
|
with gzip.open(p, "rt", encoding="utf-8") as fh:
|
|
payload = json.load(fh)
|
|
except Exception:
|
|
p.unlink(missing_ok=True)
|
|
return None
|
|
|
|
coverage_start = str(payload.get("coverage_start") or "")
|
|
coverage_end = str(payload.get("coverage_end") or "")
|
|
if not coverage_start or not coverage_end:
|
|
p.unlink(missing_ok=True)
|
|
return None
|
|
if start_date < coverage_start or end_date > coverage_end:
|
|
return None
|
|
|
|
events = payload.get("events", [])
|
|
return [
|
|
e for e in events
|
|
if start_date <= str(e.get("filing_date", ""))[:10] <= end_date
|
|
]
|
|
|
|
def put(
|
|
self,
|
|
ticker: str,
|
|
start_date: str,
|
|
end_date: str,
|
|
events: list[dict[str, Any]],
|
|
) -> None:
|
|
p = self._path(ticker)
|
|
p.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
existing_events: list[dict[str, Any]] = []
|
|
coverage_start = start_date
|
|
coverage_end = end_date
|
|
if p.exists():
|
|
try:
|
|
with gzip.open(p, "rt", encoding="utf-8") as fh:
|
|
payload = json.load(fh)
|
|
existing_events = list(payload.get("events", []))
|
|
old_start = str(payload.get("coverage_start") or "")
|
|
old_end = str(payload.get("coverage_end") or "")
|
|
if old_start:
|
|
coverage_start = min(coverage_start, old_start)
|
|
if old_end:
|
|
coverage_end = max(coverage_end, old_end)
|
|
except Exception:
|
|
p.unlink(missing_ok=True)
|
|
existing_events = []
|
|
|
|
merged: dict[tuple[str, str, str, str], dict[str, Any]] = {}
|
|
for event in existing_events + list(events):
|
|
key = (
|
|
str(event.get("filing_date", ""))[:10],
|
|
str(event.get("accession_number", "")),
|
|
str(event.get("event_type", "")),
|
|
str(event.get("item_number", "")),
|
|
)
|
|
merged[key] = event
|
|
|
|
payload = {
|
|
"coverage_start": coverage_start,
|
|
"coverage_end": coverage_end,
|
|
"events": [merged[k] for k in sorted(merged)],
|
|
}
|
|
|
|
tmp = p.with_suffix(".tmp")
|
|
try:
|
|
with gzip.open(tmp, "wt", encoding="utf-8") as fh:
|
|
json.dump(payload, fh)
|
|
os.replace(tmp, p)
|
|
except Exception:
|
|
if tmp.exists():
|
|
tmp.unlink(missing_ok=True)
|
|
raise
|
|
|
|
|
|
class AttentionEventCache:
|
|
"""Disk cache for same-day attention features keyed by ticker + event_date."""
|
|
|
|
def __init__(self, cache_dir: str = "data/cache/orb_attention") -> None:
|
|
self._root = Path(cache_dir)
|
|
|
|
def _path(self, ticker: str) -> Path:
|
|
return self._root / f"{ticker.upper()}.json.gz"
|
|
|
|
def get(self, ticker: str, event_date: str) -> dict[str, Any] | None:
|
|
p = self._path(ticker)
|
|
if not p.exists():
|
|
return None
|
|
try:
|
|
with gzip.open(p, "rt", encoding="utf-8") as fh:
|
|
payload = json.load(fh)
|
|
except Exception:
|
|
p.unlink(missing_ok=True)
|
|
return None
|
|
|
|
day_map = payload.get("days")
|
|
if not isinstance(day_map, dict):
|
|
p.unlink(missing_ok=True)
|
|
return None
|
|
data = day_map.get(event_date)
|
|
return dict(data) if isinstance(data, dict) else None
|
|
|
|
def put(self, ticker: str, event_date: str, features: dict[str, Any]) -> None:
|
|
p = self._path(ticker)
|
|
p.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
payload: dict[str, Any] = {"days": {}}
|
|
if p.exists():
|
|
try:
|
|
with gzip.open(p, "rt", encoding="utf-8") as fh:
|
|
existing = json.load(fh)
|
|
if isinstance(existing, dict) and isinstance(existing.get("days"), dict):
|
|
payload = existing
|
|
except Exception:
|
|
p.unlink(missing_ok=True)
|
|
payload = {"days": {}}
|
|
|
|
payload.setdefault("days", {})
|
|
payload["days"][event_date] = dict(features)
|
|
|
|
tmp = p.with_suffix(".tmp")
|
|
try:
|
|
with gzip.open(tmp, "wt", encoding="utf-8") as fh:
|
|
json.dump(payload, fh)
|
|
os.replace(tmp, p)
|
|
except Exception:
|
|
if tmp.exists():
|
|
tmp.unlink(missing_ok=True)
|
|
raise
|
|
|
|
|
|
class PriorEventFeatureSnapshotCache:
|
|
"""Disk snapshot cache for prior-event enrichment feature maps.
|
|
|
|
Layout:
|
|
{cache_dir}/{SNAPSHOT_ID}/manifest.json
|
|
{cache_dir}/{SNAPSHOT_ID}/{REQUEST_HASH}.json.gz
|
|
|
|
The request hash is derived from the normalized ticker list, trading days,
|
|
lookback window, and event types. This freezes the exact enrichment payload
|
|
used by a backtest once it has been materialized from the DB.
|
|
"""
|
|
|
|
_VERSION = 2
|
|
|
|
def __init__(
|
|
self,
|
|
cache_dir: str = "data/cache/orb_prior_event",
|
|
snapshot_id: str | None = None,
|
|
) -> None:
|
|
self._root = Path(cache_dir)
|
|
self._snapshot_id = self.normalize_snapshot_id(snapshot_id)
|
|
|
|
@staticmethod
|
|
def normalize_snapshot_id(snapshot_id: str | None) -> str:
|
|
raw = str(snapshot_id or "").strip().lower()
|
|
if not raw:
|
|
return "adhoc"
|
|
sanitized = "".join(
|
|
ch if ch.isalnum() or ch in {"-", "_", "."} else "-"
|
|
for ch in raw
|
|
).strip("-._")
|
|
return sanitized or "adhoc"
|
|
|
|
@property
|
|
def snapshot_id(self) -> str:
|
|
return self._snapshot_id
|
|
|
|
def _snapshot_root(self) -> Path:
|
|
return self._root / self._snapshot_id
|
|
|
|
def _manifest_path(self) -> Path:
|
|
return self._snapshot_root() / "manifest.json"
|
|
|
|
def _request_payload(
|
|
self,
|
|
tickers: list[str],
|
|
trading_days: list[str],
|
|
lookback_calendar_days: int,
|
|
event_types: tuple[str, ...],
|
|
) -> dict[str, Any]:
|
|
normalized_tickers = sorted({
|
|
str(ticker).strip().upper()
|
|
for ticker in tickers
|
|
if str(ticker).strip()
|
|
})
|
|
normalized_days = [str(day) for day in trading_days if str(day)]
|
|
normalized_event_types = sorted({
|
|
str(event_type).strip()
|
|
for event_type in event_types
|
|
if str(event_type).strip()
|
|
})
|
|
return {
|
|
"version": self._VERSION,
|
|
"tickers": normalized_tickers,
|
|
"trading_days": normalized_days,
|
|
"lookback_calendar_days": int(lookback_calendar_days),
|
|
"event_types": normalized_event_types,
|
|
}
|
|
|
|
def build_request_key(
|
|
self,
|
|
tickers: list[str],
|
|
trading_days: list[str],
|
|
lookback_calendar_days: int,
|
|
event_types: tuple[str, ...],
|
|
) -> str:
|
|
request = self._request_payload(
|
|
tickers,
|
|
trading_days,
|
|
lookback_calendar_days,
|
|
event_types,
|
|
)
|
|
return hashlib.sha256(
|
|
json.dumps(request, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
).hexdigest()
|
|
|
|
def _path_for_request(self, request: dict[str, Any]) -> Path:
|
|
digest = hashlib.sha256(
|
|
json.dumps(request, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
).hexdigest()
|
|
return self._snapshot_root() / f"{digest}.json.gz"
|
|
|
|
def _ensure_manifest(self) -> None:
|
|
root = self._snapshot_root()
|
|
root.mkdir(parents=True, exist_ok=True)
|
|
manifest_path = self._manifest_path()
|
|
if manifest_path.exists():
|
|
try:
|
|
payload = json.loads(manifest_path.read_text())
|
|
except Exception:
|
|
manifest_path.unlink(missing_ok=True)
|
|
else:
|
|
if (
|
|
isinstance(payload, dict)
|
|
and payload.get("version") == self._VERSION
|
|
and payload.get("snapshot_id") == self._snapshot_id
|
|
):
|
|
return
|
|
manifest = {
|
|
"version": self._VERSION,
|
|
"snapshot_id": self._snapshot_id,
|
|
"created_at_utc": dt.datetime.now(dt.UTC).isoformat(),
|
|
"source": "postgres_events_table",
|
|
"kind": "prior_event_feature_snapshot",
|
|
}
|
|
tmp = manifest_path.with_suffix(f".{uuid.uuid4().hex}.tmp")
|
|
try:
|
|
tmp.write_text(json.dumps(manifest, indent=2, sort_keys=True))
|
|
os.replace(tmp, manifest_path)
|
|
except Exception:
|
|
if tmp.exists():
|
|
tmp.unlink(missing_ok=True)
|
|
raise
|
|
|
|
def provenance(self) -> dict[str, Any]:
|
|
return {
|
|
"kind": "prior_event_feature_snapshot",
|
|
"snapshot_id": self._snapshot_id,
|
|
"root": str(self._snapshot_root()),
|
|
}
|
|
|
|
def get(
|
|
self,
|
|
tickers: list[str],
|
|
trading_days: list[str],
|
|
lookback_calendar_days: int,
|
|
event_types: tuple[str, ...],
|
|
) -> dict[str, dict[str, dict[str, Any]]] | None:
|
|
request = self._request_payload(
|
|
tickers,
|
|
trading_days,
|
|
lookback_calendar_days,
|
|
event_types,
|
|
)
|
|
path = self._path_for_request(request)
|
|
if not path.exists():
|
|
return None
|
|
try:
|
|
with gzip.open(path, "rt", encoding="utf-8") as fh:
|
|
payload = json.load(fh)
|
|
except Exception:
|
|
path.unlink(missing_ok=True)
|
|
return None
|
|
|
|
if (
|
|
not isinstance(payload, dict)
|
|
or payload.get("request") != request
|
|
or payload.get("version") != self._VERSION
|
|
or payload.get("snapshot_id") != self._snapshot_id
|
|
):
|
|
path.unlink(missing_ok=True)
|
|
return None
|
|
|
|
features = payload.get("features")
|
|
if not isinstance(features, dict):
|
|
path.unlink(missing_ok=True)
|
|
return None
|
|
return features
|
|
|
|
def put(
|
|
self,
|
|
tickers: list[str],
|
|
trading_days: list[str],
|
|
lookback_calendar_days: int,
|
|
event_types: tuple[str, ...],
|
|
features: dict[str, dict[str, dict[str, Any]]],
|
|
) -> None:
|
|
request = self._request_payload(
|
|
tickers,
|
|
trading_days,
|
|
lookback_calendar_days,
|
|
event_types,
|
|
)
|
|
self._ensure_manifest()
|
|
path = self._path_for_request(request)
|
|
payload = {
|
|
"version": self._VERSION,
|
|
"snapshot_id": self._snapshot_id,
|
|
"request": request,
|
|
"features": features,
|
|
}
|
|
tmp = path.with_suffix(f".{uuid.uuid4().hex}.tmp")
|
|
try:
|
|
with gzip.open(tmp, "wt", encoding="utf-8") as fh:
|
|
json.dump(payload, fh, sort_keys=True)
|
|
os.replace(tmp, path)
|
|
except Exception:
|
|
if tmp.exists():
|
|
tmp.unlink(missing_ok=True)
|
|
raise
|
|
|
|
|
|
def _event_type_weight(event_type: str) -> float:
|
|
et = event_type.lower()
|
|
if any(key in et for key in ("earnings", "results", "guidance", "regulation_fd", "material")):
|
|
return 1.0
|
|
if any(key in et for key in ("acquisition", "merger", "agreement", "contract", "asset")):
|
|
return 0.95
|
|
if any(key in et for key in ("management", "director", "officer", "analyst")):
|
|
return 0.60
|
|
return 0.75
|
|
|
|
|
|
def _build_event_feature_map(
|
|
events_by_ticker: dict[str, list[dict[str, Any]]],
|
|
start_date: str,
|
|
end_date: str,
|
|
) -> dict[str, dict[str, dict[str, Any]]]:
|
|
features: dict[str, dict[str, dict[str, Any]]] = {}
|
|
for ticker, events in events_by_ticker.items():
|
|
ticker_map: dict[str, list[dict[str, Any]]] = {}
|
|
for event in events:
|
|
filing_date = str(event.get("filing_date", ""))[:10]
|
|
if not filing_date or filing_date < start_date or filing_date > end_date:
|
|
continue
|
|
ticker_map.setdefault(filing_date, []).append(event)
|
|
|
|
if not ticker_map:
|
|
continue
|
|
|
|
day_features: dict[str, dict[str, Any]] = {}
|
|
for day, day_events in ticker_map.items():
|
|
event_types = sorted({
|
|
str(event.get("event_type", "")).strip()
|
|
for event in day_events
|
|
if str(event.get("event_type", "")).strip()
|
|
})
|
|
score = max((_event_type_weight(t) for t in event_types), default=0.0)
|
|
if len(day_events) > 1:
|
|
score = min(1.25, score + 0.10 * (len(day_events) - 1))
|
|
day_features[day] = {
|
|
"event_flag": True,
|
|
"event_count": len(day_events),
|
|
"event_types": event_types,
|
|
"event_score": round(score, 4),
|
|
}
|
|
features[ticker] = day_features
|
|
return features
|
|
|
|
|
|
def _build_attention_feature_map(
|
|
rows: dict[tuple[str, str], dict[str, Any]],
|
|
) -> dict[str, dict[str, dict[str, Any]]]:
|
|
out: dict[str, dict[str, dict[str, Any]]] = {}
|
|
for (ticker, day), features in rows.items():
|
|
out.setdefault(ticker, {})[day] = dict(features)
|
|
return out
|
|
|
|
|
|
def _extract_attention_features(payload: Any) -> dict[str, Any]:
|
|
wiki_spike = payload.wiki.spike_10d
|
|
wiki_zscore = payload.wiki.zscore_20d
|
|
article_count_3d = payload.news.article_count_3d
|
|
us_article_count_3d = payload.news.us_article_count_3d
|
|
resolver_confidence = payload.entity.resolver_confidence
|
|
return {
|
|
"attention_wiki_spike_10d": wiki_spike,
|
|
"attention_wiki_zscore_20d": wiki_zscore,
|
|
"attention_article_count_3d": article_count_3d,
|
|
"attention_us_article_count_3d": us_article_count_3d,
|
|
"attention_resolver_confidence": resolver_confidence,
|
|
}
|
|
|
|
|
|
async def fetch_filing_event_features_bulk(
|
|
tickers: list[str],
|
|
start_date: str,
|
|
end_date: str,
|
|
client: OracleClient,
|
|
cache: FilingEventCache | None = None,
|
|
concurrency: int = 6,
|
|
progress_callback: Callable[[int, int], None] | None = None,
|
|
) -> dict[str, dict[str, dict[str, Any]]]:
|
|
"""Fetch and cache same-day filing event features for a ticker universe."""
|
|
svc = FilingsService(client)
|
|
semaphore = asyncio.Semaphore(concurrency)
|
|
lock = asyncio.Lock()
|
|
events_by_ticker: dict[str, list[dict[str, Any]]] = {}
|
|
completed = 0
|
|
|
|
def _emit_progress() -> None:
|
|
if progress_callback:
|
|
progress_callback(completed, len(tickers))
|
|
|
|
async def fetch_one(ticker: str) -> None:
|
|
nonlocal completed
|
|
cached = cache.get(ticker, start_date, end_date) if cache else None
|
|
if cached is None:
|
|
async with semaphore:
|
|
try:
|
|
resp = await svc.get_filing_events(
|
|
ticker,
|
|
start_date=start_date,
|
|
end_date=end_date,
|
|
)
|
|
cached = [
|
|
{
|
|
"ticker": event.ticker,
|
|
"filing_date": event.filing_date,
|
|
"accession_number": event.accession_number,
|
|
"event_type": event.event_type,
|
|
"item_number": event.item_number,
|
|
"form_type": event.form_type,
|
|
"title": event.title,
|
|
}
|
|
for event in resp.events
|
|
]
|
|
if cache is not None:
|
|
await asyncio.to_thread(cache.put, ticker, start_date, end_date, cached)
|
|
except Exception:
|
|
cached = []
|
|
|
|
async with lock:
|
|
events_by_ticker[ticker] = list(cached)
|
|
completed += 1
|
|
_emit_progress()
|
|
|
|
await asyncio.gather(*(fetch_one(ticker) for ticker in tickers))
|
|
return _build_event_feature_map(events_by_ticker, start_date, end_date)
|
|
|
|
|
|
async def fetch_attention_features_bulk(
|
|
ticker_days: list[tuple[str, str]],
|
|
client: OracleClient,
|
|
cache: AttentionEventCache | None = None,
|
|
concurrency: int = 8,
|
|
progress_callback: Callable[[int, int], None] | None = None,
|
|
) -> dict[str, dict[str, dict[str, Any]]]:
|
|
"""Fetch same-day attention features for a set of (ticker, date) pairs."""
|
|
svc = AttentionService(client)
|
|
semaphore = asyncio.Semaphore(concurrency)
|
|
lock = asyncio.Lock()
|
|
completed = 0
|
|
rows: dict[tuple[str, str], dict[str, Any]] = {}
|
|
unique_pairs = sorted({(ticker.upper(), day) for ticker, day in ticker_days})
|
|
|
|
def _emit_progress() -> None:
|
|
if progress_callback:
|
|
progress_callback(completed, len(unique_pairs))
|
|
|
|
async def fetch_one(ticker: str, event_date: str) -> None:
|
|
nonlocal completed
|
|
cached = cache.get(ticker, event_date) if cache else None
|
|
if cached is None:
|
|
async with semaphore:
|
|
try:
|
|
payload = await svc.get_event_attention(ticker, event_date)
|
|
cached = _extract_attention_features(payload)
|
|
if cache is not None:
|
|
await asyncio.to_thread(cache.put, ticker, event_date, cached)
|
|
except Exception:
|
|
cached = {
|
|
"attention_wiki_spike_10d": None,
|
|
"attention_wiki_zscore_20d": None,
|
|
"attention_article_count_3d": 0,
|
|
"attention_us_article_count_3d": 0,
|
|
"attention_resolver_confidence": 0.0,
|
|
}
|
|
|
|
async with lock:
|
|
rows[(ticker, event_date)] = dict(cached)
|
|
completed += 1
|
|
_emit_progress()
|
|
|
|
await asyncio.gather(*(fetch_one(ticker, day) for ticker, day in unique_pairs))
|
|
return _build_attention_feature_map(rows)
|