@ -1,106 +1,109 @@
"""
Earnings Surprise service — Alpha Vantage EARNINGS API
Earnings Surprise service — SEC EDGAR XBRL
Fetches quarterly reported EPS vs estimated EPS and caches in DB .
Fetches quarterly EPS from SEC EDGAR companyfacts API and computes
QoQ surprise ( current EPS - previous quarter EPS ) .
Coverage : 2009 + for most companies . No API key required .
"""
import asyncio
import logging
import time as _time
from datetime import datetime , timezone
from typing import Dict , List , Optional , Tuple
import aiohttp
from sqlalchemy import select , desc , func
from sqlalchemy . ext . asyncio import AsyncSession
from sqlalchemy . dialects . postgresql import insert as pg_insert
from app . core . config import settings
from app . core . http_client import get_http_session
from app . models . earnings_surprise import EarningsSurprise
from app . services . sec_http_client import SECHttpClient
logger = logging . getLogger ( __name__ )
ALPHA_VANTAGE_BASE = " https://www.alphavantage.co/query "
# XBRL EPS concepts in priority order (diluted preferred over basic)
_EPS_CONCEPTS = [
" EarningsPerShareDiluted " ,
" EarningsPerShareBasic " ,
]
# Chunk size for batch insert
_CHUNK = 2000
class _RateLimiter :
""" Simple token bucket — 5 requests/minute for Alpha Vantage free tier. """
def __init__ ( self , rate : float = 5.0 / 60.0 , capacity : float = 1.0 ) :
self . _rate = rate
self . _capacity = capacity
self . _tokens = capacity
self . _last = _time . monotonic ( )
self . _lock = asyncio . Lock ( )
class EarningsService :
""" SEC EDGAR XBRL-based Earnings Surprise service. """
async def acquire ( self ) - > None :
while True :
async with self . _lock :
now = _time . monotonic ( )
self . _tokens = min ( self . _capacity , self . _tokens + ( now - self . _last ) * self . _rate )
self . _last = now
if self . _tokens > = 1.0 :
self . _tokens - = 1.0
return
wait = ( 1.0 - self . _tokens ) / self . _rate
await asyncio . sleep ( wait )
def __init__ ( self ) :
self . _http = SECHttpClient ( " Stock Oracle Earnings Service " )
# ------------------------------------------------------------------
# Fetch EPS from SEC EDGAR XBRL
# ------------------------------------------------------------------
_rate_limiter = _RateLimiter ( )
async def _fetch_eps_from_xbrl ( self , ticker : str ) - > List [ Dict ] :
""" Fetch quarterly EPS data from SEC EDGAR companyfacts XBRL API. """
cik = await self . _http . get_company_cik ( ticker )
if not cik :
raise ValueError ( f " Could not find CIK for ticker { ticker } " )
url = f " { self . _http . sec_base_data } /api/xbrl/companyfacts/CIK { cik . zfill ( 10 ) } .json "
data = await self . _http . fetch_json ( url )
class EarningsService :
""" Alpha Vantage Earnings data service. """
facts = data . get ( " facts " , { } )
us_gaap = facts . get ( " us-gaap " , { } )
# ------------------------------------------------------------------
# Fetch from Alpha Vantage
# ------------------------------------------------------------------
async def fetch_from_alpha_vantage ( self , ticker : str ) - > Optional [ Dict ] :
""" Fetch earnings data from Alpha Vantage API. """
api_key = settings . ALPHA_VANTAGE_API_KEY
if not api_key :
raise ValueError (
" ALPHA_VANTAGE_API_KEY not configured. "
" Get a free key at https://www.alphavantage.co/support/#api-key "
)
# Try each EPS concept in priority order
eps_entries = [ ]
for concept in _EPS_CONCEPTS :
if concept not in us_gaap :
continue
units = us_gaap [ concept ] . get ( " units " , { } )
usd_per_share = units . get ( " USD/shares " , [ ] )
if not usd_per_share :
continue
await _rate_limiter . acquire ( )
for entry in usd_per_share :
start = entry . get ( " start " )
end = entry . get ( " end " )
val = entry . get ( " val " )
filed = entry . get ( " filed " )
form = entry . get ( " form " , " " )
url = f " { ALPHA_VANTAGE_BASE } ?function=EARNINGS&symbol= { ticker . upper ( ) } &apikey= { api_key } "
if not end or val is None :
continue
# Only quarterly filings (10-Q) and annual (10-K)
if form not in ( " 10-Q " , " 10-K " , " 10-Q/A " , " 10-K/A " ) :
continue
# Skip annual periods for quarterly analysis (period > 100 days)
if start and end :
try :
session = await get_http_session ( )
async with session . get ( url , timeout = aiohttp . ClientTimeout ( total = 30 ) ) as resp :
if resp . status != 200 :
logger . warning ( f " Alpha Vantage HTTP { resp . status } for { ticker } " )
return None
data = await resp . json ( content_type = None )
# Alpha Vantage error responses
if " Error Message " in data :
logger . warning ( f " Alpha Vantage error for { ticker } : { data [ ' Error Message ' ] } " )
return None
if " Note " in data :
logger . warning ( f " Alpha Vantage rate limit for { ticker } : { data [ ' Note ' ] } " )
return None
if " quarterlyEarnings " not in data :
logger . warning ( f " Alpha Vantage: no quarterlyEarnings for { ticker } " )
return None
return data
except Exception as e :
logger . error ( f " Alpha Vantage fetch error for { ticker } : { e } " )
return None
s = datetime . strptime ( start , " % Y- % m- %d " )
e = datetime . strptime ( end , " % Y- % m- %d " )
if ( e - s ) . days > 100 :
continue # Annual or semi-annual period, skip
except ValueError :
continue
eps_entries . append ( {
" end " : end ,
" val " : float ( val ) ,
" filed " : filed ,
" form " : form ,
" concept " : concept ,
} )
if eps_entries :
break # Use the first concept that has data
return eps_entries
# ------------------------------------------------------------------
# Index to DB
# Index to DB with QoQ surprise
# ------------------------------------------------------------------
async def index_earnings (
self , db : AsyncSession , ticker : str , force_refresh : bool = False
) - > int :
""" Fetch earnings from Alpha Vantage and upsert into DB."""
""" Fetch EPS from SEC XBRL, compute QoQ surprise, and upsert into DB."""
ticker = ticker . upper ( )
if not force_refresh :
@ -111,48 +114,89 @@ class EarningsService:
)
if ( count . scalar ( ) or 0 ) > 0 :
return 0
else :
# Delete existing data for clean re-index
await db . execute (
EarningsSurprise . __table__ . delete ( ) . where (
EarningsSurprise . ticker == ticker
)
)
await db . flush ( )
self . _http . set_deadline ( 60.0 )
try :
eps_entries = await self . _fetch_eps_from_xbrl ( ticker )
finally :
self . _http . clear_deadline ( )
data = await self . fetch_from_alpha_vantage ( ticker )
if not data :
if not eps_entries :
logger . warning ( f " Earnings: no XBRL EPS data for { ticker } " )
return 0
rows = [ ]
for q in data . get ( " quarterlyEarnings " , [ ] ) :
fiscal_str = q . get ( " fiscalDateEnding " )
if not fiscal_str :
# Sort by date ascending
eps_entries . sort ( key = lambda x : x [ " end " ] )
# Deduplicate: entries within 15 days are the same quarter.
# Keep the one filed from 10-Q (quarterly) over 10-K (annual).
unique : List [ Dict ] = [ ]
for e in eps_entries :
if unique :
prev_date = datetime . strptime ( unique [ - 1 ] [ " end " ] , " % Y- % m- %d " )
curr_date = datetime . strptime ( e [ " end " ] , " % Y- % m- %d " )
if abs ( ( curr_date - prev_date ) . days ) < = 15 :
# Same quarter — prefer 10-Q over 10-K
if e [ " form " ] . startswith ( " 10-Q " ) and not unique [ - 1 ] [ " form " ] . startswith ( " 10-Q " ) :
unique [ - 1 ] = e
continue
unique . append ( e )
# Build rows with QoQ surprise
rows = [ ]
for i , entry in enumerate ( unique ) :
try :
fiscal_date = datetime . strptime ( fiscal_str , " % Y- % m- %d " ) . replace (
fiscal_date = datetime . strptime ( entry[ " end " ] , " % Y- % m- %d " ) . replace (
tzinfo = timezone . utc
)
except ValueError :
continue
reported_date = None
if q . get ( " reportedDate " ) :
if entry. get ( " filed " ) :
try :
reported_date = datetime . strptime (
q[ " reportedDate " ] , " % Y- % m- %d "
) . replace ( tzinfo = timezone . utc )
reported_date = datetime . strptime ( entry [ " filed " ] , " % Y- % m- %d " ) . replace (
tzinfo= timezone . utc
)
except ValueError :
pass
reported_eps = entry [ " val " ]
prev_eps = unique [ i - 1 ] [ " val " ] if i > 0 else None
surprise = None
surprise_pct = None
if prev_eps is not None :
surprise = round ( reported_eps - prev_eps , 6 )
if prev_eps != 0 :
surprise_pct = round ( ( surprise / abs ( prev_eps ) ) * 100 , 4 )
rows . append ( {
" ticker " : ticker ,
" fiscal_date_ending " : fiscal_date ,
" reported_date " : reported_date ,
" reported_eps " : _safe_float ( q . get ( " reportedEPS " ) ) ,
" estimated_eps " : _safe_float ( q . get ( " estimatedEPS " ) ) ,
" surprise " : _safe_float ( q . get ( " surprise " ) ) ,
" surprise_percentage " : _safe_float ( q . get ( " surprisePercentage " ) ) ,
" data_source " : " ALPHA_VANTAGE " ,
" reported_eps " : reported_eps ,
" estimated_eps " : prev_eps, # previous quarter as baseline
" surprise " : ,
" surprise_percentage " : surprise_pct ,
" data_source " : " SEC_XBRL " ,
} )
if not rows :
return 0
# Upsert — update existing records with latest data
stmt = pg_insert ( EarningsSurprise ) . values ( rows )
# Batch upsert
inserted = 0
for i in range ( 0 , len ( rows ) , _CHUNK ) :
chunk = rows [ i : i + _CHUNK ]
stmt = pg_insert ( EarningsSurprise ) . values ( chunk )
stmt = stmt . on_conflict_do_update (
constraint = " uq_earnings_surprise " ,
set_ = {
@ -161,14 +205,15 @@ class EarningsService:
" estimated_eps " : stmt . excluded . estimated_eps ,
" surprise " : stmt . excluded . surprise ,
" surprise_percentage " : stmt . excluded . surprise_percentage ,
" data_source " : stmt . excluded . data_source ,
} ,
)
result = await db . execute ( stmt )
inserted + = result . rowcount
await db . commit ( )
count = result . rowcount
logger . info ( f " Earnings: upserted { count } quarters for { ticker } " )
return count
logger . info ( f " Earnings: upserted { inserted } quarters for { ticker } (XBRL) " )
return inserted
# ------------------------------------------------------------------
# Query
@ -180,7 +225,6 @@ class EarningsService:
""" Get earnings surprise data. Auto-indexes if no data. """
ticker = ticker . upper ( )
# Check if data exists
count_q = await db . execute (
select ( func . count ( EarningsSurprise . id ) ) . where (
EarningsSurprise . ticker == ticker
@ -189,7 +233,6 @@ class EarningsService:
if ( count_q . scalar ( ) or 0 ) == 0 :
await self . index_earnings ( db , ticker )
# Fetch
result = await db . execute (
select ( EarningsSurprise )
. where ( EarningsSurprise . ticker == ticker )
@ -198,7 +241,7 @@ class EarningsService:
)
rows = result . scalars ( ) . all ( )
# Compute st ats
# Compute st reak
streak = 0
if rows :
first_sign = None
@ -211,7 +254,6 @@ class EarningsService:
streak + = 1 if first_sign else - 1
else :
break
# Negative streak for consecutive misses
if first_sign is False :
streak = - abs ( streak )
@ -220,12 +262,3 @@ class EarningsService:
stats = { " streak " : streak , " avg_surprise_pct " : avg_pct }
return rows , stats
def _safe_float ( val ) - > Optional [ float ] :
if val is None or val == " None " or val == " " :
return None
try :
return float ( val )
except ( ValueError , TypeError ) :
return None
I Luk Kim
5 months ago