From 971c0f053f04864217aecbccb82ac15adbdafcbd Mon Sep 17 00:00:00 2001 From: I Luk Kim Date: Wed, 25 Mar 2026 08:06:44 -0700 Subject: [PATCH] =?UTF-8?q?feat:=20Earnings=20Surprise=EB=A5=BC=20SEC=20XB?= =?UTF-8?q?RL=EB=A1=9C=20=EC=A0=84=ED=99=98=20+=20Insider/FINRA=20?= =?UTF-8?q?=EA=B3=BC=EA=B1=B0=20=EB=8D=B0=EC=9D=B4=ED=84=B0=20=ED=99=95?= =?UTF-8?q?=EC=9E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Earnings Surprise: Alpha Vantage → SEC EDGAR XBRL 전환 - companyfacts API에서 EarningsPerShareDiluted/Basic 추출 - QoQ surprise 계산 (현재 EPS - 이전 분기 EPS) - 15일 이내 중복 분기 제거 (10-Q 우선) - force_refresh 시 기존 데이터 삭제 후 재인덱싱 - API 키 불필요, 2009년~ 커버리지 2. Insider transactions: days max 1095→3650, older pages 3→10 3. FINRA: 이전 커밋에서 이미 days=3650, limit=10000 적용됨 --- app/api/v1/endpoints/earnings.py | 25 +- app/api/v1/endpoints/insider.py | 2 +- app/services/earnings_service.py | 263 +++++++++++--------- app/services/insider_transaction_service.py | 4 +- 4 files changed, 164 insertions(+), 130 deletions(-) diff --git a/app/api/v1/endpoints/earnings.py b/app/api/v1/endpoints/earnings.py index b7e356c..67a7caf 100644 --- a/app/api/v1/endpoints/earnings.py +++ b/app/api/v1/endpoints/earnings.py @@ -1,5 +1,5 @@ """ -Earnings Surprise endpoints — Alpha Vantage EPS data +Earnings Surprise endpoints — SEC EDGAR XBRL EPS data """ import logging @@ -22,20 +22,20 @@ logger = logging.getLogger("app.api.v1.earnings") response_model=EarningsSurpriseResponse, summary="Get earnings surprise history", description=( - "Quarterly earnings surprise data (reported EPS vs estimated EPS).\n\n" - "**데이터 소스**: Alpha Vantage EARNINGS API.\n" - "**필수 환경변수**: `ALPHA_VANTAGE_API_KEY` (무료 등록: https://www.alphavantage.co/support/#api-key)\n" - "**무료 tier**: 25 requests/day, 5 requests/minute.\n\n" - "**streak**: 연속 beat (양수) 또는 miss (음수) 횟수.\n" - "**beat**: surprise > 0 일 때 True." + "Quarterly EPS surprise data from SEC EDGAR XBRL.\n\n" + "**데이터 소스**: SEC EDGAR companyfacts API (무료, API 키 불필요).\n" + "**커버리지**: 2009년~ (대부분 종목). 첫 조회 시 자동 인덱싱.\n\n" + "**surprise**: 현재 분기 EPS - 이전 분기 EPS (QoQ 변화).\n" + "**estimated_eps**: 이전 분기 EPS (비교 기준).\n" + "**streak**: 연속 QoQ 증가 (양수) 또는 감소 (음수) 횟수." ), ) @with_cache(namespace="earnings:surprise", ttl=None, key_params=["symbol", "quarters"]) async def get_earnings_surprise( symbol: str, response: Response, - quarters: int = Query(8, ge=1, le=40, description="Number of recent quarters"), - force_refresh: bool = Query(False, description="Bypass cache and re-fetch from Alpha Vantage"), + quarters: int = Query(8, ge=1, le=80, description="Number of recent quarters (max 80 = ~20 years)"), + force_refresh: bool = Query(False, description="Bypass cache and re-fetch from SEC EDGAR"), db: AsyncSession = Depends(get_db), ): svc = EarningsService() @@ -44,12 +44,12 @@ async def get_earnings_surprise( try: await svc.index_earnings(db, symbol, force_refresh=True) except ValueError as e: - raise HTTPException(status_code=503, detail=str(e)) + raise HTTPException(status_code=404, detail=str(e)) try: rows, stats = await svc.get_earnings_surprise(db, ticker=symbol, quarters=quarters) except ValueError as e: - raise HTTPException(status_code=503, detail=str(e)) + raise HTTPException(status_code=404, detail=str(e)) except Exception as e: logger.error(f"Earnings surprise error for {symbol}: {e}") raise HTTPException(status_code=502, detail=f"Failed to fetch earnings data: {e}") @@ -62,8 +62,9 @@ async def get_earnings_surprise( streak=stats["streak"], avg_surprise_pct=stats["avg_surprise_pct"], metadata={ - "data_source": "ALPHA_VANTAGE", + "data_source": "SEC_XBRL", "quarters_requested": quarters, "quarters_returned": len(entries), + "note": "surprise = current_eps - previous_quarter_eps (QoQ change)", }, ) diff --git a/app/api/v1/endpoints/insider.py b/app/api/v1/endpoints/insider.py index 17b39c9..5cf4ad7 100644 --- a/app/api/v1/endpoints/insider.py +++ b/app/api/v1/endpoints/insider.py @@ -37,7 +37,7 @@ logger = logging.getLogger("app.api.v1.insider") async def get_insider_transactions( symbol: str, response: Response, - days: int = Query(90, ge=1, le=1095, description="Days to look back (max 3 years)"), + days: int = Query(90, ge=1, le=3650, description="Days to look back (max ~10 years)"), transaction_type: Optional[str] = Query(None, description="Filter: P=Purchase, S=Sale, A=Award, M=Exercise"), insider_title: Optional[str] = Query(None, description="Filter by title keyword (e.g., CEO, CFO, Director)"), limit: int = Query(50, ge=1, le=500, description="Max entries to return"), diff --git a/app/services/earnings_service.py b/app/services/earnings_service.py index 3d8de5f..5d6eaa2 100644 --- a/app/services/earnings_service.py +++ b/app/services/earnings_service.py @@ -1,106 +1,109 @@ """ -Earnings Surprise service — Alpha Vantage EARNINGS API +Earnings Surprise service — SEC EDGAR XBRL -Fetches quarterly reported EPS vs estimated EPS and caches in DB. +Fetches quarterly EPS from SEC EDGAR companyfacts API and computes +QoQ surprise (current EPS - previous quarter EPS). +Coverage: 2009+ for most companies. No API key required. """ -import asyncio import logging -import time as _time from datetime import datetime, timezone from typing import Dict, List, Optional, Tuple -import aiohttp from sqlalchemy import select, desc, func from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.dialects.postgresql import insert as pg_insert -from app.core.config import settings -from app.core.http_client import get_http_session from app.models.earnings_surprise import EarningsSurprise +from app.services.sec_http_client import SECHttpClient logger = logging.getLogger(__name__) -ALPHA_VANTAGE_BASE = "https://www.alphavantage.co/query" +# XBRL EPS concepts in priority order (diluted preferred over basic) +_EPS_CONCEPTS = [ + "EarningsPerShareDiluted", + "EarningsPerShareBasic", +] - -class _RateLimiter: - """Simple token bucket — 5 requests/minute for Alpha Vantage free tier.""" - - def __init__(self, rate: float = 5.0 / 60.0, capacity: float = 1.0): - self._rate = rate - self._capacity = capacity - self._tokens = capacity - self._last = _time.monotonic() - self._lock = asyncio.Lock() - - async def acquire(self) -> None: - while True: - async with self._lock: - now = _time.monotonic() - self._tokens = min(self._capacity, self._tokens + (now - self._last) * self._rate) - self._last = now - if self._tokens >= 1.0: - self._tokens -= 1.0 - return - wait = (1.0 - self._tokens) / self._rate - await asyncio.sleep(wait) - - -_rate_limiter = _RateLimiter() +# Chunk size for batch insert +_CHUNK = 2000 class EarningsService: - """Alpha Vantage Earnings data service.""" + """SEC EDGAR XBRL-based Earnings Surprise service.""" + + def __init__(self): + self._http = SECHttpClient("Stock Oracle Earnings Service") # ------------------------------------------------------------------ - # Fetch from Alpha Vantage + # Fetch EPS from SEC EDGAR XBRL # ------------------------------------------------------------------ - async def fetch_from_alpha_vantage(self, ticker: str) -> Optional[Dict]: - """Fetch earnings data from Alpha Vantage API.""" - api_key = settings.ALPHA_VANTAGE_API_KEY - if not api_key: - raise ValueError( - "ALPHA_VANTAGE_API_KEY not configured. " - "Get a free key at https://www.alphavantage.co/support/#api-key" - ) + async def _fetch_eps_from_xbrl(self, ticker: str) -> List[Dict]: + """Fetch quarterly EPS data from SEC EDGAR companyfacts XBRL API.""" + cik = await self._http.get_company_cik(ticker) + if not cik: + raise ValueError(f"Could not find CIK for ticker {ticker}") - await _rate_limiter.acquire() + url = f"{self._http.sec_base_data}/api/xbrl/companyfacts/CIK{cik.zfill(10)}.json" + data = await self._http.fetch_json(url) - url = f"{ALPHA_VANTAGE_BASE}?function=EARNINGS&symbol={ticker.upper()}&apikey={api_key}" - try: - session = await get_http_session() - async with session.get(url, timeout=aiohttp.ClientTimeout(total=30)) as resp: - if resp.status != 200: - logger.warning(f"Alpha Vantage HTTP {resp.status} for {ticker}") - return None - data = await resp.json(content_type=None) - - # Alpha Vantage error responses - if "Error Message" in data: - logger.warning(f"Alpha Vantage error for {ticker}: {data['Error Message']}") - return None - if "Note" in data: - logger.warning(f"Alpha Vantage rate limit for {ticker}: {data['Note']}") - return None - if "quarterlyEarnings" not in data: - logger.warning(f"Alpha Vantage: no quarterlyEarnings for {ticker}") - return None - - return data - except Exception as e: - logger.error(f"Alpha Vantage fetch error for {ticker}: {e}") - return None + facts = data.get("facts", {}) + us_gaap = facts.get("us-gaap", {}) + + # Try each EPS concept in priority order + eps_entries = [] + for concept in _EPS_CONCEPTS: + if concept not in us_gaap: + continue + units = us_gaap[concept].get("units", {}) + usd_per_share = units.get("USD/shares", []) + if not usd_per_share: + continue + + for entry in usd_per_share: + start = entry.get("start") + end = entry.get("end") + val = entry.get("val") + filed = entry.get("filed") + form = entry.get("form", "") + + if not end or val is None: + continue + # Only quarterly filings (10-Q) and annual (10-K) + if form not in ("10-Q", "10-K", "10-Q/A", "10-K/A"): + continue + # Skip annual periods for quarterly analysis (period > 100 days) + if start and end: + try: + s = datetime.strptime(start, "%Y-%m-%d") + e = datetime.strptime(end, "%Y-%m-%d") + if (e - s).days > 100: + continue # Annual or semi-annual period, skip + except ValueError: + continue + + eps_entries.append({ + "end": end, + "val": float(val), + "filed": filed, + "form": form, + "concept": concept, + }) + + if eps_entries: + break # Use the first concept that has data + + return eps_entries # ------------------------------------------------------------------ - # Index to DB + # Index to DB with QoQ surprise # ------------------------------------------------------------------ async def index_earnings( self, db: AsyncSession, ticker: str, force_refresh: bool = False ) -> int: - """Fetch earnings from Alpha Vantage and upsert into DB.""" + """Fetch EPS from SEC XBRL, compute QoQ surprise, and upsert into DB.""" ticker = ticker.upper() if not force_refresh: @@ -111,64 +114,106 @@ class EarningsService: ) if (count.scalar() or 0) > 0: return 0 + else: + # Delete existing data for clean re-index + await db.execute( + EarningsSurprise.__table__.delete().where( + EarningsSurprise.ticker == ticker + ) + ) + await db.flush() - data = await self.fetch_from_alpha_vantage(ticker) - if not data: + self._http.set_deadline(60.0) + try: + eps_entries = await self._fetch_eps_from_xbrl(ticker) + finally: + self._http.clear_deadline() + + if not eps_entries: + logger.warning(f"Earnings: no XBRL EPS data for {ticker}") return 0 + # Sort by date ascending + eps_entries.sort(key=lambda x: x["end"]) + + # Deduplicate: entries within 15 days are the same quarter. + # Keep the one filed from 10-Q (quarterly) over 10-K (annual). + unique: List[Dict] = [] + for e in eps_entries: + if unique: + prev_date = datetime.strptime(unique[-1]["end"], "%Y-%m-%d") + curr_date = datetime.strptime(e["end"], "%Y-%m-%d") + if abs((curr_date - prev_date).days) <= 15: + # Same quarter — prefer 10-Q over 10-K + if e["form"].startswith("10-Q") and not unique[-1]["form"].startswith("10-Q"): + unique[-1] = e + continue + unique.append(e) + + # Build rows with QoQ surprise rows = [] - for q in data.get("quarterlyEarnings", []): - fiscal_str = q.get("fiscalDateEnding") - if not fiscal_str: - continue + for i, entry in enumerate(unique): try: - fiscal_date = datetime.strptime(fiscal_str, "%Y-%m-%d").replace( + fiscal_date = datetime.strptime(entry["end"], "%Y-%m-%d").replace( tzinfo=timezone.utc ) except ValueError: continue reported_date = None - if q.get("reportedDate"): + if entry.get("filed"): try: - reported_date = datetime.strptime( - q["reportedDate"], "%Y-%m-%d" - ).replace(tzinfo=timezone.utc) + reported_date = datetime.strptime(entry["filed"], "%Y-%m-%d").replace( + tzinfo=timezone.utc + ) except ValueError: pass + reported_eps = entry["val"] + prev_eps = unique[i - 1]["val"] if i > 0 else None + surprise = None + surprise_pct = None + if prev_eps is not None: + surprise = round(reported_eps - prev_eps, 6) + if prev_eps != 0: + surprise_pct = round((surprise / abs(prev_eps)) * 100, 4) + rows.append({ "ticker": ticker, "fiscal_date_ending": fiscal_date, "reported_date": reported_date, - "reported_eps": _safe_float(q.get("reportedEPS")), - "estimated_eps": _safe_float(q.get("estimatedEPS")), - "surprise": _safe_float(q.get("surprise")), - "surprise_percentage": _safe_float(q.get("surprisePercentage")), - "data_source": "ALPHA_VANTAGE", + "reported_eps": reported_eps, + "estimated_eps": prev_eps, # previous quarter as baseline + "surprise": surprise, + "surprise_percentage": surprise_pct, + "data_source": "SEC_XBRL", }) if not rows: return 0 - # Upsert — update existing records with latest data - stmt = pg_insert(EarningsSurprise).values(rows) - stmt = stmt.on_conflict_do_update( - constraint="uq_earnings_surprise", - set_={ - "reported_date": stmt.excluded.reported_date, - "reported_eps": stmt.excluded.reported_eps, - "estimated_eps": stmt.excluded.estimated_eps, - "surprise": stmt.excluded.surprise, - "surprise_percentage": stmt.excluded.surprise_percentage, - }, - ) - result = await db.execute(stmt) + # Batch upsert + inserted = 0 + for i in range(0, len(rows), _CHUNK): + chunk = rows[i:i + _CHUNK] + stmt = pg_insert(EarningsSurprise).values(chunk) + stmt = stmt.on_conflict_do_update( + constraint="uq_earnings_surprise", + set_={ + "reported_date": stmt.excluded.reported_date, + "reported_eps": stmt.excluded.reported_eps, + "estimated_eps": stmt.excluded.estimated_eps, + "surprise": stmt.excluded.surprise, + "surprise_percentage": stmt.excluded.surprise_percentage, + "data_source": stmt.excluded.data_source, + }, + ) + result = await db.execute(stmt) + inserted += result.rowcount await db.commit() - count = result.rowcount - logger.info(f"Earnings: upserted {count} quarters for {ticker}") - return count + logger.info(f"Earnings: upserted {inserted} quarters for {ticker} (XBRL)") + return inserted # ------------------------------------------------------------------ # Query @@ -180,7 +225,6 @@ class EarningsService: """Get earnings surprise data. Auto-indexes if no data.""" ticker = ticker.upper() - # Check if data exists count_q = await db.execute( select(func.count(EarningsSurprise.id)).where( EarningsSurprise.ticker == ticker @@ -189,7 +233,6 @@ class EarningsService: if (count_q.scalar() or 0) == 0: await self.index_earnings(db, ticker) - # Fetch result = await db.execute( select(EarningsSurprise) .where(EarningsSurprise.ticker == ticker) @@ -198,7 +241,7 @@ class EarningsService: ) rows = result.scalars().all() - # Compute stats + # Compute streak streak = 0 if rows: first_sign = None @@ -211,7 +254,6 @@ class EarningsService: streak += 1 if first_sign else -1 else: break - # Negative streak for consecutive misses if first_sign is False: streak = -abs(streak) @@ -220,12 +262,3 @@ class EarningsService: stats = {"streak": streak, "avg_surprise_pct": avg_pct} return rows, stats - - -def _safe_float(val) -> Optional[float]: - if val is None or val == "None" or val == "": - return None - try: - return float(val) - except (ValueError, TypeError): - return None diff --git a/app/services/insider_transaction_service.py b/app/services/insider_transaction_service.py index fefdf62..189c9b7 100644 --- a/app/services/insider_transaction_service.py +++ b/app/services/insider_transaction_service.py @@ -83,8 +83,8 @@ class InsiderTransactionService: recent = data.get("filings", {}).get("recent", {}) scan_block(recent) - # Older filing pages (up to 3) - for meta in (data.get("filings", {}).get("files", []) or [])[:3]: + # Older filing pages (up to 10 — covers 10+ years of Form 4s) + for meta in (data.get("filings", {}).get("files", []) or [])[:10]: name = meta.get("name") if not name: continue