|
|
"""
|
|
|
PIT Dividend Calendar service — yfinance-plus
|
|
|
|
|
|
PIT rules:
|
|
|
- Historical dividends: as_of_date = ex_dividend_date - 30 days
|
|
|
(approximation: dividends are typically declared 2-4 weeks before ex-date)
|
|
|
- Upcoming/future dividends: as_of_date = ingestion timestamp
|
|
|
(captures when we first observed the announcement)
|
|
|
|
|
|
Special dividend detection:
|
|
|
- Heuristic: amount >= 2.5 × median of most recent 12 payments
|
|
|
- Applied per-ticker at fetch time; stored in dividend_type column
|
|
|
"""
|
|
|
|
|
|
import logging
|
|
|
import math
|
|
|
from collections import Counter
|
|
|
from datetime import date, datetime, timedelta, timezone
|
|
|
from typing import Dict, List, Optional, Tuple
|
|
|
|
|
|
from sqlalchemy import select, and_, func, desc, delete
|
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
|
|
|
|
|
from app.models.dividend_calendar import DividendCalendar
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
_CHUNK = 2000 # floor(32767 / 16 columns) = 2047, rounded down
|
|
|
|
|
|
|
|
|
class DividendService:
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
# yfinance data extraction (synchronous — called via asyncio.to_thread)
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
|
def _fetch_dividends_from_yfinance(self, ticker: str) -> List[Dict]:
|
|
|
"""Fetch dividend data from yfinance-plus. Returns list of row dicts."""
|
|
|
import warnings
|
|
|
warnings.filterwarnings("ignore", category=DeprecationWarning)
|
|
|
warnings.filterwarnings("ignore", category=FutureWarning)
|
|
|
|
|
|
try:
|
|
|
from yfinance_plus import Ticker
|
|
|
except ImportError:
|
|
|
import yfinance as yf
|
|
|
Ticker = yf.Ticker
|
|
|
|
|
|
ticker = ticker.upper()
|
|
|
t = Ticker(ticker)
|
|
|
now_utc = datetime.now(timezone.utc)
|
|
|
rows: List[Dict] = []
|
|
|
|
|
|
# ---- 1. Historical dividends ----
|
|
|
try:
|
|
|
divs = t.dividends # pandas Series: DatetimeIndex -> float
|
|
|
if divs is not None and not divs.empty:
|
|
|
for dt_idx, amount in divs.items():
|
|
|
amount_f = _safe_float(amount)
|
|
|
if amount_f is None or amount_f <= 0:
|
|
|
continue
|
|
|
|
|
|
ex_date = _normalize_ex_date(dt_idx)
|
|
|
if ex_date is None:
|
|
|
continue
|
|
|
|
|
|
# PIT: dividends are typically declared 2-4 weeks before ex-date.
|
|
|
# Without declaration_date from yfinance, approximate with -30 days.
|
|
|
as_of = ex_date - timedelta(days=30)
|
|
|
|
|
|
rows.append({
|
|
|
"ticker": ticker,
|
|
|
"ex_dividend_date": ex_date,
|
|
|
"amount": round(amount_f, 6),
|
|
|
"declaration_date": None,
|
|
|
"record_date": None,
|
|
|
"payment_date": None,
|
|
|
"currency": "USD",
|
|
|
"dividend_type": "regular",
|
|
|
"frequency": None,
|
|
|
"as_of_date": as_of,
|
|
|
"source": "yfinance",
|
|
|
"source_file_date": None,
|
|
|
})
|
|
|
except Exception as e:
|
|
|
logger.warning(f"Dividend: t.dividends failed for {ticker}: {e}")
|
|
|
|
|
|
# ---- 2. Upcoming dividend from calendar + info (fetched once) ----
|
|
|
# Fetch info once and reuse in step 3 to avoid duplicate API calls.
|
|
|
info: Dict = {}
|
|
|
try:
|
|
|
info = t.info or {}
|
|
|
except Exception:
|
|
|
pass
|
|
|
|
|
|
try:
|
|
|
cal = t.calendar
|
|
|
if cal is not None and isinstance(cal, dict):
|
|
|
ex_date_val = cal.get("Ex-Dividend Date")
|
|
|
div_date_val = cal.get("Dividend Date")
|
|
|
|
|
|
if ex_date_val is not None:
|
|
|
ex_date = _normalize_ex_date(ex_date_val)
|
|
|
if ex_date and ex_date > now_utc:
|
|
|
last_div = _safe_float(info.get("lastDividendValue"))
|
|
|
div_rate = _safe_float(info.get("dividendRate"))
|
|
|
amount = last_div or (div_rate / 4 if div_rate else None)
|
|
|
|
|
|
if amount and amount > 0:
|
|
|
payment_date = _to_utc_datetime(div_date_val)
|
|
|
rows.append({
|
|
|
"ticker": ticker,
|
|
|
"ex_dividend_date": ex_date,
|
|
|
"amount": round(amount, 6),
|
|
|
"declaration_date": None,
|
|
|
"record_date": None,
|
|
|
"payment_date": payment_date,
|
|
|
"currency": "USD",
|
|
|
"dividend_type": "regular",
|
|
|
"frequency": None,
|
|
|
"as_of_date": now_utc, # PIT: known NOW (ingestion time)
|
|
|
"source": "yfinance",
|
|
|
"source_file_date": None,
|
|
|
})
|
|
|
except Exception as e:
|
|
|
logger.warning(f"Dividend: t.calendar failed for {ticker}: {e}")
|
|
|
|
|
|
# ---- 3. Enrich with frequency (info already fetched above) ----
|
|
|
try:
|
|
|
freq_str = _infer_frequency(info) or _infer_frequency_from_history(rows)
|
|
|
if freq_str:
|
|
|
for row in rows:
|
|
|
row["frequency"] = freq_str
|
|
|
except Exception:
|
|
|
pass
|
|
|
|
|
|
# ---- 4. Flag special dividends ----
|
|
|
_flag_special_dividends(rows)
|
|
|
|
|
|
return rows
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
# Ingest a single ticker to DB
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
|
async def index_dividends(
|
|
|
self,
|
|
|
db: AsyncSession,
|
|
|
ticker: str,
|
|
|
force_refresh: bool = False,
|
|
|
) -> int:
|
|
|
"""Fetch from yfinance and upsert to DB. Returns number of rows upserted.
|
|
|
|
|
|
When force_refresh=True: fetch first, then delete+insert in one transaction
|
|
|
to avoid data loss if the fetch fails.
|
|
|
"""
|
|
|
ticker = ticker.upper()
|
|
|
|
|
|
if not force_refresh:
|
|
|
count_q = await db.execute(
|
|
|
select(func.count(DividendCalendar.id)).where(
|
|
|
DividendCalendar.ticker == ticker
|
|
|
)
|
|
|
)
|
|
|
if (count_q.scalar() or 0) > 0:
|
|
|
return 0
|
|
|
|
|
|
# Fetch from yfinance BEFORE deleting existing rows.
|
|
|
# This prevents data loss if the fetch fails.
|
|
|
import asyncio
|
|
|
try:
|
|
|
raw = await asyncio.to_thread(self._fetch_dividends_from_yfinance, ticker)
|
|
|
except Exception as e:
|
|
|
logger.error(f"Dividend: yfinance fetch failed for {ticker}: {e}")
|
|
|
raise ValueError(f"Could not fetch dividend data for {ticker}: {e}")
|
|
|
|
|
|
if force_refresh:
|
|
|
# Safe to delete now that we have fresh data
|
|
|
await db.execute(
|
|
|
delete(DividendCalendar).where(DividendCalendar.ticker == ticker)
|
|
|
)
|
|
|
|
|
|
if not raw:
|
|
|
if force_refresh:
|
|
|
await db.commit()
|
|
|
logger.info(f"Dividend: no data returned by yfinance for {ticker}")
|
|
|
return 0
|
|
|
|
|
|
inserted = 0
|
|
|
for i in range(0, len(raw), _CHUNK):
|
|
|
chunk = raw[i:i + _CHUNK]
|
|
|
stmt = pg_insert(DividendCalendar).values(chunk)
|
|
|
stmt = stmt.on_conflict_do_update(
|
|
|
constraint="uq_dividend_calendar",
|
|
|
set_={
|
|
|
"amount": stmt.excluded.amount,
|
|
|
"declaration_date": stmt.excluded.declaration_date,
|
|
|
"record_date": stmt.excluded.record_date,
|
|
|
"payment_date": stmt.excluded.payment_date,
|
|
|
"currency": stmt.excluded.currency,
|
|
|
"dividend_type": stmt.excluded.dividend_type,
|
|
|
"frequency": stmt.excluded.frequency,
|
|
|
"source_file_date": stmt.excluded.source_file_date,
|
|
|
"updated_at": func.now(),
|
|
|
},
|
|
|
)
|
|
|
result = await db.execute(stmt)
|
|
|
inserted += result.rowcount
|
|
|
|
|
|
await db.commit()
|
|
|
logger.info(f"Dividend: upserted {inserted} records for {ticker}")
|
|
|
return inserted
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
# Bulk ingest (admin backfill)
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
|
async def bulk_ingest(
|
|
|
self,
|
|
|
db: AsyncSession,
|
|
|
symbols: List[str],
|
|
|
force_refresh: bool = False,
|
|
|
) -> Dict:
|
|
|
"""Ingest dividends for multiple symbols sequentially. Returns summary."""
|
|
|
total_upserted = 0
|
|
|
failed: List[str] = []
|
|
|
|
|
|
for sym in symbols:
|
|
|
try:
|
|
|
count = await self.index_dividends(db, sym, force_refresh=force_refresh)
|
|
|
total_upserted += count
|
|
|
except Exception as e:
|
|
|
logger.error(f"Dividend: bulk ingest failed for {sym}: {e}")
|
|
|
failed.append(sym)
|
|
|
|
|
|
return {
|
|
|
"symbols_processed": len(symbols),
|
|
|
"total_records_upserted": total_upserted,
|
|
|
"failed_symbols": failed,
|
|
|
}
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
# PIT query — upcoming dividends
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
|
async def get_upcoming_dividends(
|
|
|
self,
|
|
|
db: AsyncSession,
|
|
|
as_of_date: datetime,
|
|
|
from_ex_date: datetime,
|
|
|
to_ex_date: datetime,
|
|
|
symbols: Optional[List[str]] = None,
|
|
|
limit: int = 500,
|
|
|
) -> Tuple[List[DividendCalendar], int]:
|
|
|
"""
|
|
|
PIT upcoming dividends query.
|
|
|
|
|
|
Uses DISTINCT ON (ticker, ex_dividend_date) + ORDER BY as_of_date DESC
|
|
|
to return the latest-known revision for each dividend event as of as_of_date.
|
|
|
|
|
|
When specific symbols are requested but not yet in the DB, auto-indexes
|
|
|
them from yfinance (same behaviour as get_dividend_history).
|
|
|
|
|
|
SQL equivalent:
|
|
|
SELECT DISTINCT ON (ticker, ex_dividend_date) *
|
|
|
FROM dividend_calendar
|
|
|
WHERE as_of_date <= :as_of_date
|
|
|
AND ex_dividend_date BETWEEN :from_ex_date AND :to_ex_date
|
|
|
ORDER BY ticker, ex_dividend_date, as_of_date DESC
|
|
|
"""
|
|
|
# Auto-index any requested symbols not yet in the DB (deduped)
|
|
|
if symbols:
|
|
|
upper_syms = list(dict.fromkeys(s.upper() for s in symbols)) # dedup, preserve order
|
|
|
existing_q = await db.execute(
|
|
|
select(DividendCalendar.ticker.distinct()).where(
|
|
|
DividendCalendar.ticker.in_(upper_syms)
|
|
|
)
|
|
|
)
|
|
|
existing = {r for r in existing_q.scalars().all()}
|
|
|
missing = [s for s in upper_syms if s not in existing]
|
|
|
for sym in missing:
|
|
|
try:
|
|
|
await self.index_dividends(db, sym)
|
|
|
except Exception as e:
|
|
|
logger.warning(f"Dividend: auto-index failed for {sym}: {e}")
|
|
|
|
|
|
conditions = [
|
|
|
DividendCalendar.as_of_date <= as_of_date,
|
|
|
DividendCalendar.ex_dividend_date >= from_ex_date,
|
|
|
DividendCalendar.ex_dividend_date <= to_ex_date,
|
|
|
]
|
|
|
if symbols:
|
|
|
conditions.append(DividendCalendar.ticker.in_([s.upper() for s in symbols]))
|
|
|
|
|
|
# DISTINCT ON via SQLAlchemy .distinct(col1, col2) — PostgreSQL only
|
|
|
pit_stmt = (
|
|
|
select(DividendCalendar)
|
|
|
.where(and_(*conditions))
|
|
|
.order_by(
|
|
|
DividendCalendar.ticker,
|
|
|
DividendCalendar.ex_dividend_date,
|
|
|
desc(DividendCalendar.as_of_date),
|
|
|
)
|
|
|
.distinct(DividendCalendar.ticker, DividendCalendar.ex_dividend_date)
|
|
|
)
|
|
|
|
|
|
# Total count via subquery
|
|
|
count_stmt = select(func.count()).select_from(pit_stmt.subquery())
|
|
|
total = (await db.execute(count_stmt)).scalar() or 0
|
|
|
|
|
|
# Fetch with limit
|
|
|
result = await db.execute(pit_stmt.limit(limit))
|
|
|
rows = result.scalars().all()
|
|
|
|
|
|
return rows, total
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
# History query (per symbol)
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
|
async def get_dividend_history(
|
|
|
self,
|
|
|
db: AsyncSession,
|
|
|
ticker: str,
|
|
|
limit: int = 100,
|
|
|
) -> Tuple[List[DividendCalendar], int, Optional[float]]:
|
|
|
"""
|
|
|
Dividend history for a single symbol.
|
|
|
|
|
|
Returns latest-known revision per ex_date (desc), total count,
|
|
|
and trailing 12-month dividend sum for yield estimation.
|
|
|
Auto-indexes from yfinance if no data exists.
|
|
|
"""
|
|
|
ticker = ticker.upper()
|
|
|
|
|
|
count_q = await db.execute(
|
|
|
select(func.count(DividendCalendar.id)).where(
|
|
|
DividendCalendar.ticker == ticker
|
|
|
)
|
|
|
)
|
|
|
if (count_q.scalar() or 0) == 0:
|
|
|
await self.index_dividends(db, ticker)
|
|
|
|
|
|
now_utc = datetime.now(timezone.utc)
|
|
|
|
|
|
# PIT query: latest revision per ex_date as of now
|
|
|
pit_stmt = (
|
|
|
select(DividendCalendar)
|
|
|
.where(
|
|
|
and_(
|
|
|
DividendCalendar.ticker == ticker,
|
|
|
DividendCalendar.as_of_date <= now_utc,
|
|
|
)
|
|
|
)
|
|
|
.order_by(
|
|
|
DividendCalendar.ticker,
|
|
|
DividendCalendar.ex_dividend_date,
|
|
|
desc(DividendCalendar.as_of_date),
|
|
|
)
|
|
|
.distinct(DividendCalendar.ticker, DividendCalendar.ex_dividend_date)
|
|
|
)
|
|
|
|
|
|
result = await db.execute(pit_stmt)
|
|
|
all_rows = result.scalars().all()
|
|
|
|
|
|
# Sort by ex_date desc for the response
|
|
|
all_rows_sorted = sorted(all_rows, key=lambda r: r.ex_dividend_date, reverse=True)
|
|
|
total = len(all_rows_sorted)
|
|
|
rows = all_rows_sorted[:limit]
|
|
|
|
|
|
# TTM: sum of amounts with ex_date in last 365 days
|
|
|
ttm_cutoff = now_utc - timedelta(days=365)
|
|
|
ttm_sum = sum(
|
|
|
r.amount for r in all_rows_sorted
|
|
|
if r.ex_dividend_date >= ttm_cutoff
|
|
|
)
|
|
|
annual_yield = round(ttm_sum, 4) if ttm_sum > 0 else None
|
|
|
|
|
|
return rows, total, annual_yield
|
|
|
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
# Utility helpers
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
|
def _safe_float(val) -> Optional[float]:
|
|
|
"""Convert to float, returning None for None/NaN/±Inf."""
|
|
|
if val is None:
|
|
|
return None
|
|
|
try:
|
|
|
f = float(val)
|
|
|
if math.isnan(f) or math.isinf(f):
|
|
|
return None
|
|
|
return f
|
|
|
except (ValueError, TypeError):
|
|
|
return None
|
|
|
|
|
|
|
|
|
def _to_utc_datetime(val) -> Optional[datetime]:
|
|
|
"""Convert date/datetime/Timestamp/str to UTC-aware datetime."""
|
|
|
if val is None:
|
|
|
return None
|
|
|
if isinstance(val, datetime):
|
|
|
return val.astimezone(timezone.utc) if val.tzinfo else val.replace(tzinfo=timezone.utc)
|
|
|
if hasattr(val, "to_pydatetime"):
|
|
|
return _to_utc_datetime(val.to_pydatetime())
|
|
|
if isinstance(val, date) and not isinstance(val, datetime):
|
|
|
return datetime(val.year, val.month, val.day, tzinfo=timezone.utc)
|
|
|
if isinstance(val, str):
|
|
|
try:
|
|
|
dt = datetime.fromisoformat(val)
|
|
|
return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)
|
|
|
except ValueError:
|
|
|
return None
|
|
|
return None
|
|
|
|
|
|
|
|
|
def _normalize_ex_date(val) -> Optional[datetime]:
|
|
|
"""Convert an ex-dividend date to UTC midnight.
|
|
|
|
|
|
yfinance returns dates in US/Eastern time (e.g. 2025-02-10 05:00:00+00 which
|
|
|
is 2025-02-10 00:00 EST). We normalize to UTC midnight of the Eastern calendar
|
|
|
date to prevent timezone artifacts from creating duplicate rows.
|
|
|
"""
|
|
|
dt = _to_utc_datetime(val)
|
|
|
if dt is None:
|
|
|
return None
|
|
|
try:
|
|
|
import zoneinfo
|
|
|
eastern = zoneinfo.ZoneInfo("America/New_York")
|
|
|
local_dt = dt.astimezone(eastern)
|
|
|
return datetime(local_dt.year, local_dt.month, local_dt.day, tzinfo=timezone.utc)
|
|
|
except Exception:
|
|
|
return datetime(dt.year, dt.month, dt.day, tzinfo=timezone.utc)
|
|
|
|
|
|
|
|
|
def _infer_frequency(info: dict) -> Optional[str]:
|
|
|
"""Infer dividend frequency from yfinance info dict."""
|
|
|
freq_hint = info.get("dividendFrequency")
|
|
|
if freq_hint:
|
|
|
fh = str(freq_hint).lower()
|
|
|
mapping = {"1": "annual", "2": "semi-annual", "4": "quarterly", "12": "monthly"}
|
|
|
return mapping.get(fh, fh)
|
|
|
|
|
|
rate = _safe_float(info.get("dividendRate"))
|
|
|
last_val = _safe_float(info.get("lastDividendValue"))
|
|
|
if rate and last_val and last_val > 0:
|
|
|
ratio = rate / last_val
|
|
|
if 3.5 <= ratio <= 4.5:
|
|
|
return "quarterly"
|
|
|
if 1.8 <= ratio <= 2.2:
|
|
|
return "semi-annual"
|
|
|
if 0.8 <= ratio <= 1.2:
|
|
|
return "annual"
|
|
|
if 11.0 <= ratio <= 13.0:
|
|
|
return "monthly"
|
|
|
return None
|
|
|
|
|
|
|
|
|
def _infer_frequency_from_history(rows: List[Dict]) -> Optional[str]:
|
|
|
"""Infer payment frequency by counting payments per year in recent history.
|
|
|
|
|
|
Uses the most recent 3 years of data to avoid frequency changes in older
|
|
|
history skewing the estimate. Only considers regular dividends (called
|
|
|
before _flag_special_dividends, so all rows are "regular" at this point).
|
|
|
"""
|
|
|
if len(rows) < 3:
|
|
|
return None
|
|
|
|
|
|
# Sort by ex_date, use last 3 years
|
|
|
dated = sorted(
|
|
|
[r for r in rows if r["amount"] > 0],
|
|
|
key=lambda r: r["ex_dividend_date"],
|
|
|
)
|
|
|
if not dated:
|
|
|
return None
|
|
|
|
|
|
cutoff = dated[-1]["ex_dividend_date"] - timedelta(days=3 * 365)
|
|
|
recent = [r for r in dated if r["ex_dividend_date"] >= cutoff]
|
|
|
if len(recent) < 3:
|
|
|
return None
|
|
|
|
|
|
# Count payments per calendar year
|
|
|
years: Counter = Counter()
|
|
|
for r in recent:
|
|
|
dt = r["ex_dividend_date"]
|
|
|
year = dt.year if hasattr(dt, "year") else dt
|
|
|
years[year] += 1
|
|
|
|
|
|
if not years:
|
|
|
return None
|
|
|
|
|
|
# Median payments-per-year to reduce skew from partial years at boundaries
|
|
|
counts = sorted(years.values())
|
|
|
median_count = counts[len(counts) // 2]
|
|
|
|
|
|
if median_count >= 10:
|
|
|
return "monthly"
|
|
|
if median_count >= 3:
|
|
|
return "quarterly"
|
|
|
if median_count >= 2:
|
|
|
return "semi-annual"
|
|
|
if median_count == 1:
|
|
|
return "annual"
|
|
|
return None
|
|
|
|
|
|
|
|
|
def _flag_special_dividends(rows: List[Dict]) -> None:
|
|
|
"""Heuristic: flag dividends that are clear outliers relative to recent history.
|
|
|
|
|
|
Uses the median of the most recent 12 payments as the baseline, so long-term
|
|
|
dividend growers (MSFT, JNJ) are not incorrectly flagged. Requires at least
|
|
|
4 recent payments to avoid false positives.
|
|
|
"""
|
|
|
if len(rows) < 4:
|
|
|
return
|
|
|
|
|
|
dated = sorted(
|
|
|
[r for r in rows if r["amount"] > 0],
|
|
|
key=lambda r: r["ex_dividend_date"],
|
|
|
)
|
|
|
if not dated:
|
|
|
return
|
|
|
|
|
|
baseline_window = dated[-12:]
|
|
|
if len(baseline_window) < 4:
|
|
|
return
|
|
|
|
|
|
amounts = sorted(r["amount"] for r in baseline_window)
|
|
|
median = amounts[len(amounts) // 2]
|
|
|
if median <= 0:
|
|
|
return
|
|
|
|
|
|
threshold = median * 2.5
|
|
|
for row in rows:
|
|
|
if row["amount"] >= threshold:
|
|
|
row["dividend_type"] = "special"
|