""" PIT Dividend Calendar service — yfinance-plus PIT rules: - Historical dividends: as_of_date = ex_dividend_date - 30 days (approximation: dividends are typically declared 2-4 weeks before ex-date) - Upcoming/future dividends: as_of_date = ingestion timestamp (captures when we first observed the announcement) Special dividend detection: - Heuristic: amount >= 2.5 × median of most recent 12 payments - Applied per-ticker at fetch time; stored in dividend_type column """ import logging import math from collections import Counter from datetime import date, datetime, timedelta, timezone from typing import Dict, List, Optional, Tuple from sqlalchemy import select, and_, func, desc, delete from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.dialects.postgresql import insert as pg_insert from app.models.dividend_calendar import DividendCalendar logger = logging.getLogger(__name__) _CHUNK = 2000 # floor(32767 / 16 columns) = 2047, rounded down class DividendService: # ------------------------------------------------------------------ # yfinance data extraction (synchronous — called via asyncio.to_thread) # ------------------------------------------------------------------ def _fetch_dividends_from_yfinance(self, ticker: str) -> List[Dict]: """Fetch dividend data from yfinance-plus. Returns list of row dicts.""" import warnings warnings.filterwarnings("ignore", category=DeprecationWarning) warnings.filterwarnings("ignore", category=FutureWarning) try: from yfinance_plus import Ticker except ImportError: import yfinance as yf Ticker = yf.Ticker ticker = ticker.upper() t = Ticker(ticker) now_utc = datetime.now(timezone.utc) rows: List[Dict] = [] # ---- 1. Historical dividends ---- try: divs = t.dividends # pandas Series: DatetimeIndex -> float if divs is not None and not divs.empty: for dt_idx, amount in divs.items(): amount_f = _safe_float(amount) if amount_f is None or amount_f <= 0: continue ex_date = _normalize_ex_date(dt_idx) if ex_date is None: continue # PIT: dividends are typically declared 2-4 weeks before ex-date. # Without declaration_date from yfinance, approximate with -30 days. as_of = ex_date - timedelta(days=30) rows.append({ "ticker": ticker, "ex_dividend_date": ex_date, "amount": round(amount_f, 6), "declaration_date": None, "record_date": None, "payment_date": None, "currency": "USD", "dividend_type": "regular", "frequency": None, "as_of_date": as_of, "source": "yfinance", "source_file_date": None, }) except Exception as e: logger.warning(f"Dividend: t.dividends failed for {ticker}: {e}") # ---- 2. Upcoming dividend from calendar + info (fetched once) ---- # Fetch info once and reuse in step 3 to avoid duplicate API calls. info: Dict = {} try: info = t.info or {} except Exception: pass try: cal = t.calendar if cal is not None and isinstance(cal, dict): ex_date_val = cal.get("Ex-Dividend Date") div_date_val = cal.get("Dividend Date") if ex_date_val is not None: ex_date = _normalize_ex_date(ex_date_val) if ex_date and ex_date > now_utc: last_div = _safe_float(info.get("lastDividendValue")) div_rate = _safe_float(info.get("dividendRate")) amount = last_div or (div_rate / 4 if div_rate else None) if amount and amount > 0: payment_date = _to_utc_datetime(div_date_val) rows.append({ "ticker": ticker, "ex_dividend_date": ex_date, "amount": round(amount, 6), "declaration_date": None, "record_date": None, "payment_date": payment_date, "currency": "USD", "dividend_type": "regular", "frequency": None, "as_of_date": now_utc, # PIT: known NOW (ingestion time) "source": "yfinance", "source_file_date": None, }) except Exception as e: logger.warning(f"Dividend: t.calendar failed for {ticker}: {e}") # ---- 3. Enrich with frequency (info already fetched above) ---- try: freq_str = _infer_frequency(info) or _infer_frequency_from_history(rows) if freq_str: for row in rows: row["frequency"] = freq_str except Exception: pass # ---- 4. Flag special dividends ---- _flag_special_dividends(rows) return rows # ------------------------------------------------------------------ # Ingest a single ticker to DB # ------------------------------------------------------------------ async def index_dividends( self, db: AsyncSession, ticker: str, force_refresh: bool = False, ) -> int: """Fetch from yfinance and upsert to DB. Returns number of rows upserted. When force_refresh=True: fetch first, then delete+insert in one transaction to avoid data loss if the fetch fails. """ ticker = ticker.upper() if not force_refresh: count_q = await db.execute( select(func.count(DividendCalendar.id)).where( DividendCalendar.ticker == ticker ) ) if (count_q.scalar() or 0) > 0: return 0 # Fetch from yfinance BEFORE deleting existing rows. # This prevents data loss if the fetch fails. import asyncio try: raw = await asyncio.to_thread(self._fetch_dividends_from_yfinance, ticker) except Exception as e: logger.error(f"Dividend: yfinance fetch failed for {ticker}: {e}") raise ValueError(f"Could not fetch dividend data for {ticker}: {e}") if force_refresh: # Safe to delete now that we have fresh data await db.execute( delete(DividendCalendar).where(DividendCalendar.ticker == ticker) ) if not raw: if force_refresh: await db.commit() logger.info(f"Dividend: no data returned by yfinance for {ticker}") return 0 inserted = 0 for i in range(0, len(raw), _CHUNK): chunk = raw[i:i + _CHUNK] stmt = pg_insert(DividendCalendar).values(chunk) stmt = stmt.on_conflict_do_update( constraint="uq_dividend_calendar", set_={ "amount": stmt.excluded.amount, "declaration_date": stmt.excluded.declaration_date, "record_date": stmt.excluded.record_date, "payment_date": stmt.excluded.payment_date, "currency": stmt.excluded.currency, "dividend_type": stmt.excluded.dividend_type, "frequency": stmt.excluded.frequency, "source_file_date": stmt.excluded.source_file_date, "updated_at": func.now(), }, ) result = await db.execute(stmt) inserted += result.rowcount await db.commit() logger.info(f"Dividend: upserted {inserted} records for {ticker}") return inserted # ------------------------------------------------------------------ # Bulk ingest (admin backfill) # ------------------------------------------------------------------ async def bulk_ingest( self, db: AsyncSession, symbols: List[str], force_refresh: bool = False, ) -> Dict: """Ingest dividends for multiple symbols sequentially. Returns summary.""" total_upserted = 0 failed: List[str] = [] for sym in symbols: try: count = await self.index_dividends(db, sym, force_refresh=force_refresh) total_upserted += count except Exception as e: logger.error(f"Dividend: bulk ingest failed for {sym}: {e}") failed.append(sym) return { "symbols_processed": len(symbols), "total_records_upserted": total_upserted, "failed_symbols": failed, } # ------------------------------------------------------------------ # PIT query — upcoming dividends # ------------------------------------------------------------------ async def get_upcoming_dividends( self, db: AsyncSession, as_of_date: datetime, from_ex_date: datetime, to_ex_date: datetime, symbols: Optional[List[str]] = None, limit: int = 500, ) -> Tuple[List[DividendCalendar], int]: """ PIT upcoming dividends query. Uses DISTINCT ON (ticker, ex_dividend_date) + ORDER BY as_of_date DESC to return the latest-known revision for each dividend event as of as_of_date. When specific symbols are requested but not yet in the DB, auto-indexes them from yfinance (same behaviour as get_dividend_history). SQL equivalent: SELECT DISTINCT ON (ticker, ex_dividend_date) * FROM dividend_calendar WHERE as_of_date <= :as_of_date AND ex_dividend_date BETWEEN :from_ex_date AND :to_ex_date ORDER BY ticker, ex_dividend_date, as_of_date DESC """ # Auto-index any requested symbols not yet in the DB (deduped) if symbols: upper_syms = list(dict.fromkeys(s.upper() for s in symbols)) # dedup, preserve order existing_q = await db.execute( select(DividendCalendar.ticker.distinct()).where( DividendCalendar.ticker.in_(upper_syms) ) ) existing = {r for r in existing_q.scalars().all()} missing = [s for s in upper_syms if s not in existing] for sym in missing: try: await self.index_dividends(db, sym) except Exception as e: logger.warning(f"Dividend: auto-index failed for {sym}: {e}") conditions = [ DividendCalendar.as_of_date <= as_of_date, DividendCalendar.ex_dividend_date >= from_ex_date, DividendCalendar.ex_dividend_date <= to_ex_date, ] if symbols: conditions.append(DividendCalendar.ticker.in_([s.upper() for s in symbols])) # DISTINCT ON via SQLAlchemy .distinct(col1, col2) — PostgreSQL only pit_stmt = ( select(DividendCalendar) .where(and_(*conditions)) .order_by( DividendCalendar.ticker, DividendCalendar.ex_dividend_date, desc(DividendCalendar.as_of_date), ) .distinct(DividendCalendar.ticker, DividendCalendar.ex_dividend_date) ) # Total count via subquery count_stmt = select(func.count()).select_from(pit_stmt.subquery()) total = (await db.execute(count_stmt)).scalar() or 0 # Fetch with limit result = await db.execute(pit_stmt.limit(limit)) rows = result.scalars().all() return rows, total # ------------------------------------------------------------------ # History query (per symbol) # ------------------------------------------------------------------ async def get_dividend_history( self, db: AsyncSession, ticker: str, limit: int = 100, ) -> Tuple[List[DividendCalendar], int, Optional[float]]: """ Dividend history for a single symbol. Returns latest-known revision per ex_date (desc), total count, and trailing 12-month dividend sum for yield estimation. Auto-indexes from yfinance if no data exists. """ ticker = ticker.upper() count_q = await db.execute( select(func.count(DividendCalendar.id)).where( DividendCalendar.ticker == ticker ) ) if (count_q.scalar() or 0) == 0: await self.index_dividends(db, ticker) now_utc = datetime.now(timezone.utc) # PIT query: latest revision per ex_date as of now pit_stmt = ( select(DividendCalendar) .where( and_( DividendCalendar.ticker == ticker, DividendCalendar.as_of_date <= now_utc, ) ) .order_by( DividendCalendar.ticker, DividendCalendar.ex_dividend_date, desc(DividendCalendar.as_of_date), ) .distinct(DividendCalendar.ticker, DividendCalendar.ex_dividend_date) ) result = await db.execute(pit_stmt) all_rows = result.scalars().all() # Sort by ex_date desc for the response all_rows_sorted = sorted(all_rows, key=lambda r: r.ex_dividend_date, reverse=True) total = len(all_rows_sorted) rows = all_rows_sorted[:limit] # TTM: sum of amounts with ex_date in last 365 days ttm_cutoff = now_utc - timedelta(days=365) ttm_sum = sum( r.amount for r in all_rows_sorted if r.ex_dividend_date >= ttm_cutoff ) annual_yield = round(ttm_sum, 4) if ttm_sum > 0 else None return rows, total, annual_yield # ------------------------------------------------------------------ # Utility helpers # ------------------------------------------------------------------ def _safe_float(val) -> Optional[float]: """Convert to float, returning None for None/NaN/±Inf.""" if val is None: return None try: f = float(val) if math.isnan(f) or math.isinf(f): return None return f except (ValueError, TypeError): return None def _to_utc_datetime(val) -> Optional[datetime]: """Convert date/datetime/Timestamp/str to UTC-aware datetime.""" if val is None: return None if isinstance(val, datetime): return val.astimezone(timezone.utc) if val.tzinfo else val.replace(tzinfo=timezone.utc) if hasattr(val, "to_pydatetime"): return _to_utc_datetime(val.to_pydatetime()) if isinstance(val, date) and not isinstance(val, datetime): return datetime(val.year, val.month, val.day, tzinfo=timezone.utc) if isinstance(val, str): try: dt = datetime.fromisoformat(val) return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc) except ValueError: return None return None def _normalize_ex_date(val) -> Optional[datetime]: """Convert an ex-dividend date to UTC midnight. yfinance returns dates in US/Eastern time (e.g. 2025-02-10 05:00:00+00 which is 2025-02-10 00:00 EST). We normalize to UTC midnight of the Eastern calendar date to prevent timezone artifacts from creating duplicate rows. """ dt = _to_utc_datetime(val) if dt is None: return None try: import zoneinfo eastern = zoneinfo.ZoneInfo("America/New_York") local_dt = dt.astimezone(eastern) return datetime(local_dt.year, local_dt.month, local_dt.day, tzinfo=timezone.utc) except Exception: return datetime(dt.year, dt.month, dt.day, tzinfo=timezone.utc) def _infer_frequency(info: dict) -> Optional[str]: """Infer dividend frequency from yfinance info dict.""" freq_hint = info.get("dividendFrequency") if freq_hint: fh = str(freq_hint).lower() mapping = {"1": "annual", "2": "semi-annual", "4": "quarterly", "12": "monthly"} return mapping.get(fh, fh) rate = _safe_float(info.get("dividendRate")) last_val = _safe_float(info.get("lastDividendValue")) if rate and last_val and last_val > 0: ratio = rate / last_val if 3.5 <= ratio <= 4.5: return "quarterly" if 1.8 <= ratio <= 2.2: return "semi-annual" if 0.8 <= ratio <= 1.2: return "annual" if 11.0 <= ratio <= 13.0: return "monthly" return None def _infer_frequency_from_history(rows: List[Dict]) -> Optional[str]: """Infer payment frequency by counting payments per year in recent history. Uses the most recent 3 years of data to avoid frequency changes in older history skewing the estimate. Only considers regular dividends (called before _flag_special_dividends, so all rows are "regular" at this point). """ if len(rows) < 3: return None # Sort by ex_date, use last 3 years dated = sorted( [r for r in rows if r["amount"] > 0], key=lambda r: r["ex_dividend_date"], ) if not dated: return None cutoff = dated[-1]["ex_dividend_date"] - timedelta(days=3 * 365) recent = [r for r in dated if r["ex_dividend_date"] >= cutoff] if len(recent) < 3: return None # Count payments per calendar year years: Counter = Counter() for r in recent: dt = r["ex_dividend_date"] year = dt.year if hasattr(dt, "year") else dt years[year] += 1 if not years: return None # Median payments-per-year to reduce skew from partial years at boundaries counts = sorted(years.values()) median_count = counts[len(counts) // 2] if median_count >= 10: return "monthly" if median_count >= 3: return "quarterly" if median_count >= 2: return "semi-annual" if median_count == 1: return "annual" return None def _flag_special_dividends(rows: List[Dict]) -> None: """Heuristic: flag dividends that are clear outliers relative to recent history. Uses the median of the most recent 12 payments as the baseline, so long-term dividend growers (MSFT, JNJ) are not incorrectly flagged. Requires at least 4 recent payments to avoid false positives. """ if len(rows) < 4: return dated = sorted( [r for r in rows if r["amount"] > 0], key=lambda r: r["ex_dividend_date"], ) if not dated: return baseline_window = dated[-12:] if len(baseline_window) < 4: return amounts = sorted(r["amount"] for r in baseline_window) median = amounts[len(amounts) // 2] if median <= 0: return threshold = median * 2.5 for row in rows: if row["amount"] >= threshold: row["dividend_type"] = "special"