@ -46,6 +46,23 @@ class ScreenerService:
" price_to_book " : " pricebook " ,
}
# Maps API sort_by names to keys on the parsed-stock dict. Used only in
# min_dollar_volume mode, where results are assembled across multiple
# Yahoo pages and must be re-sorted server-side.
POST_SORT_KEY = {
" market_cap " : " market_cap " ,
" volume " : " volume " ,
" avg_volume " : " avg_volume_3m " ,
" price " : " price " ,
" pe_ratio " : " pe_ratio " ,
" change_percent " : " change_percent " ,
" name " : " name " ,
" eps " : " eps_ttm " ,
" dividend_yield " : " dividend_yield " ,
" forward_pe " : " forward_pe " ,
" price_to_book " : " price_to_book " ,
}
def _build_query (
self ,
market_cap_min : Optional [ float ] ,
@ -160,6 +177,47 @@ class ScreenerService:
import yfinance_plus as yf
return yf . screen ( preset , offset = offset , count = count )
# Yahoo's screen() max page is 250; cap total internal fetch depth so a
# broad query (e.g. low market_cap_min) can't issue unbounded requests.
_COLLECT_PAGE = 250
_COLLECT_MAX_PAGES = 24 # up to 6000 rows
async def _collect_all_quotes ( self , query , sort_field : str , sort_asc : bool ) :
""" Page through Yahoo screen() until the full match set is collected.
Returns ( quotes , yahoo_total , truncated ) . Used only by the
min_dollar_volume path , which must post - filter the complete set .
"""
loop = asyncio . get_event_loop ( )
seen : dict = { }
offset = 0
yahoo_total = None
pages = 0
while pages < self . _COLLECT_MAX_PAGES :
raw = await loop . run_in_executor (
None , self . _screen_sync , query , offset ,
self . _COLLECT_PAGE , sort_field , sort_asc ,
)
qs = raw . get ( ' quotes ' , [ ] ) or [ ]
if yahoo_total is None :
yahoo_total = raw . get ( ' total ' ) or raw . get ( ' count ' ) or 0
for q in qs :
sym = q . get ( ' symbol ' )
if sym and sym not in seen :
seen [ sym ] = q
pages + = 1
offset + = self . _COLLECT_PAGE
if not qs or len ( qs ) < self . _COLLECT_PAGE :
break
if yahoo_total and offset > = yahoo_total :
break
truncated = bool (
pages > = self . _COLLECT_MAX_PAGES
and yahoo_total
and offset < yahoo_total
)
return list ( seen . values ( ) ) , ( yahoo_total or len ( seen ) ) , truncated
async def screen_preset ( self , preset : str , page : int = 1 , page_size : int = 25 ) - > dict :
""" Fetch a Yahoo Finance predefined screener (e.g. day_gainers). """
start_time = time . time ( )
@ -173,7 +231,10 @@ class ScreenerService:
)
quotes = raw . get ( ' quotes ' , [ ] )
total_available = raw . get ( ' count ' ) or raw . get ( ' total ' ) or len ( quotes )
# Yahoo returns the full match count under 'total'; 'count' is only the
# number of rows in THIS page (== size). Prefer 'total' so total_pages
# reflects the real result set, not a single page.
total_available = raw . get ( ' total ' ) or raw . get ( ' count ' ) or len ( quotes )
total_pages = max ( 1 , ( total_available + page_size - 1 ) / / page_size )
return {
@ -196,6 +257,7 @@ class ScreenerService:
market_cap_max : Optional [ float ] = None ,
exchange : Optional [ str ] = None ,
min_avg_volume : Optional [ int ] = None ,
min_dollar_volume : Optional [ float ] = None ,
exclude_types : Optional [ str ] = None ,
sector : Optional [ str ] = None ,
pe_min : Optional [ float ] = None ,
@ -207,17 +269,31 @@ class ScreenerService:
sort_by : str = " market_cap " ,
sort_ascending : bool = False ,
) - > dict :
""" Screen stocks with the given filters and return paginated results. """
""" Screen stocks with the given filters and return paginated results.
` ` min_dollar_volume ` ` is opt - in . When it is None the behaviour is
byte - for - byte identical to before this parameter existed ( single Yahoo
page , Yahoo applies ` ` min_avg_volume ` ` ) . When set , the share - count
` ` min_avg_volume ` ` filter is intentionally NOT pushed to Yahoo — that
would drop high - priced , low - share - volume names ( BLK , KLAC , … ) at the
source before dollar volume can be evaluated — and the complete match
set is fetched and post - filtered on price × averageDailyVolume3Month .
"""
start_time = time . time ( )
page_size = max ( 1 , min ( page_size , 250 ) )
page = max ( 1 , page )
dollar_mode = min_dollar_volume is not None
# In dollar-volume mode the share-count avg-volume gate is replaced by
# the dollar-volume gate, so it must not be sent to Yahoo.
effective_min_avg_volume = None if dollar_mode else min_avg_volume
query = self . _build_query (
market_cap_min = market_cap_min ,
market_cap_max = market_cap_max ,
exchange = exchange ,
min_avg_volume = min_avg_volume ,
min_avg_volume = effective_ min_avg_volume,
sector = sector ,
pe_min = pe_min ,
pe_max = pe_max ,
@ -226,8 +302,17 @@ class ScreenerService:
)
sort_field = self . SORT_FIELD_MAP . get ( sort_by , " intradaymarketcap " )
offset = ( page - 1 ) * page_size
# Post-filter: remove non-equity types if requested
exclude_type_set = set ( )
if exclude_types :
exclude_type_set = { t . strip ( ) . upper ( ) for t in exclude_types . split ( ' , ' ) }
extra_meta : dict = { }
if not dollar_mode :
# ---- Unchanged legacy path (zero regression when opt-in is off) ----
offset = ( page - 1 ) * page_size
loop = asyncio . get_event_loop ( )
raw = await loop . run_in_executor (
None ,
@ -240,13 +325,10 @@ class ScreenerService:
)
quotes = raw . get ( ' quotes ' , [ ] )
# yfinance may return total count under 'count' or 'total'
total_available = raw . get ( ' count ' ) or raw . get ( ' total ' ) or len ( quotes )
# Post-filter: remove non-equity types if requested
exclude_type_set = set ( )
if exclude_types :
exclude_type_set = { t . strip ( ) . upper ( ) for t in exclude_types . split ( ' , ' ) }
# Yahoo's 'total' is the full match count; 'count' is only this
# page's row count. Prefer 'total' so total_pages is correct and
# clients that paginate by total_pages don't stop after page 1.
total_available = raw . get ( ' total ' ) or raw . get ( ' count ' ) or len ( quotes )
stocks = [ ]
for quote in quotes :
@ -255,6 +337,51 @@ class ScreenerService:
if qt in exclude_type_set :
continue
stocks . append ( self . _parse_quote ( quote ) )
else :
# ---- Opt-in dollar-volume path: fetch full set, post-filter ----
all_quotes , _yahoo_total , truncated = await self . _collect_all_quotes (
query , sort_field , sort_ascending
)
filtered = [ ]
for quote in all_quotes :
if exclude_type_set :
qt = ( quote . get ( ' quoteType ' ) or ' ' ) . upper ( )
if qt in exclude_type_set :
continue
price = quote . get ( ' regularMarketPrice ' )
avg_vol = quote . get ( ' averageDailyVolume3Month ' )
if price is None or avg_vol is None :
continue
if price * avg_vol < min_dollar_volume :
continue
filtered . append ( self . _parse_quote ( quote ) )
sort_key = self . POST_SORT_KEY . get ( sort_by , " market_cap " )
# Partition so rows missing the sort field are always last,
# regardless of sort direction, and so str/num keys never mix.
present = [ s for s in filtered if s . get ( sort_key ) is not None ]
missing = [ s for s in filtered if s . get ( sort_key ) is None ]
present . sort ( key = lambda s : s . get ( sort_key ) , reverse = not sort_ascending )
filtered = present + missing
total_available = len ( filtered )
start = ( page - 1 ) * page_size
stocks = filtered [ start : start + page_size ]
extra_meta [ ' dollar_volume_mode ' ] = True
extra_meta [ ' fetched_universe ' ] = len ( all_quotes )
if min_avg_volume is not None :
extra_meta [ ' note_min_avg_volume ' ] = (
' min_avg_volume ignored because min_dollar_volume is set '
' (dollar volume replaces the share-count liquidity gate) '
)
if truncated :
extra_meta [ ' truncated ' ] = True
extra_meta [ ' note_truncated ' ] = (
' Yahoo result set exceeded internal fetch cap; widen '
' market_cap_min to narrow the universe for completeness '
)
query_time = time . time ( ) - start_time
total_pages = max ( 1 , ( total_available + page_size - 1 ) / / page_size )
@ -266,8 +393,10 @@ class ScreenerService:
filters_applied [ ' market_cap_max ' ] = market_cap_max
if exchange :
filters_applied [ ' exchange ' ] = exchange
if min_avg_volume is not None :
if min_avg_volume is not None and not dollar_mode :
filters_applied [ ' min_avg_volume ' ] = min_avg_volume
if min_dollar_volume is not None :
filters_applied [ ' min_dollar_volume ' ] = min_dollar_volume
if exclude_types :
filters_applied [ ' exclude_types ' ] = exclude_types
if sector :
@ -295,6 +424,7 @@ class ScreenerService:
' sort_ascending ' : sort_ascending ,
' source ' : ' yfinance_screen ' ,
' note ' : ' sector/industry not included in per-stock response (Yahoo API limitation) ' ,
* * extra_meta ,
} ,
}
I Luk Kim
3 months ago