|
|
"""
|
|
|
Stock Screener Service
|
|
|
|
|
|
Uses yfinance EquityQuery + screen() for real-time stock filtering.
|
|
|
Results are passed through directly (no DB storage) with Redis caching at the API layer.
|
|
|
"""
|
|
|
import time
|
|
|
import asyncio
|
|
|
import logging
|
|
|
from typing import Optional
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
|
class ScreenerService:
|
|
|
# Maps user-facing exchange names to yfinance exchange codes
|
|
|
EXCHANGE_MAP = {
|
|
|
"NYSE": ["NYQ"],
|
|
|
"NASDAQ": ["NMS", "NGM", "NCM"],
|
|
|
"AMEX": ["ASE"],
|
|
|
"NYSE_ARCA": ["PCX"],
|
|
|
}
|
|
|
|
|
|
# Maps raw yfinance exchange codes back to friendly names
|
|
|
REVERSE_EXCHANGE_MAP = {
|
|
|
"NYQ": "NYSE",
|
|
|
"NMS": "NASDAQ",
|
|
|
"NGM": "NASDAQ",
|
|
|
"NCM": "NASDAQ",
|
|
|
"ASE": "AMEX",
|
|
|
"PCX": "NYSE_ARCA",
|
|
|
}
|
|
|
|
|
|
# Maps API sort_by parameter names to yfinance sort field names
|
|
|
SORT_FIELD_MAP = {
|
|
|
"market_cap": "intradaymarketcap",
|
|
|
"volume": "dayvolume",
|
|
|
"avg_volume": "avgdailyvol3m",
|
|
|
"price": "intradayprice",
|
|
|
"pe_ratio": "peratio.lasttwelvemonths",
|
|
|
"change_percent": "percentchange",
|
|
|
"name": "companyshortname",
|
|
|
"eps": "epstrailingtwelvemonths",
|
|
|
"dividend_yield": "trailingannualdividendyield",
|
|
|
"forward_pe": "forwardpricetoearnings",
|
|
|
"price_to_book": "pricebook",
|
|
|
}
|
|
|
|
|
|
# Maps API sort_by names to keys on the parsed-stock dict. Used only in
|
|
|
# min_dollar_volume mode, where results are assembled across multiple
|
|
|
# Yahoo pages and must be re-sorted server-side.
|
|
|
POST_SORT_KEY = {
|
|
|
"market_cap": "market_cap",
|
|
|
"volume": "volume",
|
|
|
"avg_volume": "avg_volume_3m",
|
|
|
"price": "price",
|
|
|
"pe_ratio": "pe_ratio",
|
|
|
"change_percent": "change_percent",
|
|
|
"name": "name",
|
|
|
"eps": "eps_ttm",
|
|
|
"dividend_yield": "dividend_yield",
|
|
|
"forward_pe": "forward_pe",
|
|
|
"price_to_book": "price_to_book",
|
|
|
}
|
|
|
|
|
|
def _build_query(
|
|
|
self,
|
|
|
market_cap_min: Optional[float],
|
|
|
market_cap_max: Optional[float],
|
|
|
exchange: Optional[str],
|
|
|
min_avg_volume: Optional[int],
|
|
|
sector: Optional[str],
|
|
|
pe_min: Optional[float],
|
|
|
pe_max: Optional[float],
|
|
|
price_min: Optional[float],
|
|
|
price_max: Optional[float],
|
|
|
):
|
|
|
"""Build EquityQuery from filter parameters."""
|
|
|
try:
|
|
|
from yfinance import EquityQuery
|
|
|
except ImportError:
|
|
|
raise RuntimeError("yfinance is not installed")
|
|
|
|
|
|
conditions = []
|
|
|
|
|
|
# Market cap filter
|
|
|
if market_cap_min is not None and market_cap_max is not None:
|
|
|
conditions.append(EquityQuery('btwn', ['intradaymarketcap', market_cap_min, market_cap_max]))
|
|
|
elif market_cap_min is not None:
|
|
|
conditions.append(EquityQuery('gt', ['intradaymarketcap', market_cap_min]))
|
|
|
elif market_cap_max is not None:
|
|
|
conditions.append(EquityQuery('lt', ['intradaymarketcap', market_cap_max]))
|
|
|
|
|
|
# Exchange filter
|
|
|
if exchange:
|
|
|
exchange_codes = []
|
|
|
for ex in exchange.split(','):
|
|
|
ex = ex.strip().upper()
|
|
|
codes = self.EXCHANGE_MAP.get(ex, [ex])
|
|
|
exchange_codes.extend(codes)
|
|
|
if exchange_codes:
|
|
|
# is-in syntax: field name + values all in one list (not nested)
|
|
|
conditions.append(EquityQuery('is-in', ['exchange', *exchange_codes]))
|
|
|
else:
|
|
|
# Default to US market when no exchange specified
|
|
|
conditions.append(EquityQuery('eq', ['region', 'us']))
|
|
|
|
|
|
# Average volume filter
|
|
|
if min_avg_volume is not None:
|
|
|
conditions.append(EquityQuery('gt', ['avgdailyvol3m', min_avg_volume]))
|
|
|
|
|
|
# Sector filter (filtering works, but sector won't appear in response per Yahoo API limits)
|
|
|
if sector:
|
|
|
conditions.append(EquityQuery('eq', ['sector', sector]))
|
|
|
|
|
|
# PE ratio filter
|
|
|
if pe_min is not None and pe_max is not None:
|
|
|
conditions.append(EquityQuery('btwn', ['peratio.lasttwelvemonths', pe_min, pe_max]))
|
|
|
elif pe_min is not None:
|
|
|
conditions.append(EquityQuery('gt', ['peratio.lasttwelvemonths', pe_min]))
|
|
|
elif pe_max is not None:
|
|
|
conditions.append(EquityQuery('lt', ['peratio.lasttwelvemonths', pe_max]))
|
|
|
|
|
|
# Price filter
|
|
|
if price_min is not None and price_max is not None:
|
|
|
conditions.append(EquityQuery('btwn', ['intradayprice', price_min, price_max]))
|
|
|
elif price_min is not None:
|
|
|
conditions.append(EquityQuery('gt', ['intradayprice', price_min]))
|
|
|
elif price_max is not None:
|
|
|
conditions.append(EquityQuery('lt', ['intradayprice', price_max]))
|
|
|
|
|
|
if not conditions:
|
|
|
return EquityQuery('eq', ['region', 'us'])
|
|
|
if len(conditions) == 1:
|
|
|
return conditions[0]
|
|
|
return EquityQuery('and', conditions)
|
|
|
|
|
|
def _parse_quote(self, quote: dict) -> dict:
|
|
|
"""Parse a raw yfinance quote dict into our response format."""
|
|
|
exchange_code = quote.get('exchange', '')
|
|
|
exchange_friendly = self.REVERSE_EXCHANGE_MAP.get(exchange_code, exchange_code)
|
|
|
|
|
|
# Volume fields may be float from yfinance; cast to int if present
|
|
|
volume = quote.get('regularMarketVolume')
|
|
|
avg_volume_3m = quote.get('averageDailyVolume3Month')
|
|
|
|
|
|
return {
|
|
|
'symbol': quote.get('symbol', ''),
|
|
|
'name': quote.get('shortName') or quote.get('longName'),
|
|
|
'exchange': exchange_friendly,
|
|
|
'exchange_code': exchange_code,
|
|
|
'quote_type': quote.get('quoteType'),
|
|
|
'market_cap': quote.get('marketCap'),
|
|
|
'price': quote.get('regularMarketPrice'),
|
|
|
'change_percent': quote.get('regularMarketChangePercent'),
|
|
|
'volume': int(volume) if volume is not None else None,
|
|
|
'avg_volume_3m': int(avg_volume_3m) if avg_volume_3m is not None else None,
|
|
|
'shares_outstanding': quote.get('sharesOutstanding'),
|
|
|
'pe_ratio': quote.get('trailingPE'),
|
|
|
'forward_pe': quote.get('forwardPE'),
|
|
|
'eps_ttm': quote.get('epsTrailingTwelveMonths'),
|
|
|
'dividend_yield': quote.get('trailingAnnualDividendYield'),
|
|
|
'fifty_two_week_high': quote.get('fiftyTwoWeekHigh'),
|
|
|
'fifty_two_week_low': quote.get('fiftyTwoWeekLow'),
|
|
|
'analyst_rating': quote.get('averageAnalystRating'),
|
|
|
'book_value': quote.get('bookValue'),
|
|
|
'price_to_book': quote.get('priceToBook'),
|
|
|
}
|
|
|
|
|
|
def _screen_sync(self, query, offset: int, size: int, sort_field: str, sort_asc: bool) -> dict:
|
|
|
"""Synchronous yfinance screen() call — must run in executor."""
|
|
|
import yfinance_plus as yf
|
|
|
return yf.screen(query, offset=offset, size=size, sortField=sort_field, sortAsc=sort_asc)
|
|
|
|
|
|
def _screen_preset_sync(self, preset: str, offset: int, count: int) -> dict:
|
|
|
"""Synchronous preset screen() call — must run in executor."""
|
|
|
import yfinance_plus as yf
|
|
|
return yf.screen(preset, offset=offset, count=count)
|
|
|
|
|
|
# Yahoo's screen() max page is 250; cap total internal fetch depth so a
|
|
|
# broad query (e.g. low market_cap_min) can't issue unbounded requests.
|
|
|
_COLLECT_PAGE = 250
|
|
|
_COLLECT_MAX_PAGES = 24 # up to 6000 rows
|
|
|
|
|
|
async def _collect_all_quotes(self, query, sort_field: str, sort_asc: bool):
|
|
|
"""Page through Yahoo screen() until the full match set is collected.
|
|
|
|
|
|
Returns (quotes, yahoo_total, truncated). Used only by the
|
|
|
min_dollar_volume path, which must post-filter the complete set.
|
|
|
"""
|
|
|
loop = asyncio.get_event_loop()
|
|
|
seen: dict = {}
|
|
|
offset = 0
|
|
|
yahoo_total = None
|
|
|
pages = 0
|
|
|
while pages < self._COLLECT_MAX_PAGES:
|
|
|
raw = await loop.run_in_executor(
|
|
|
None, self._screen_sync, query, offset,
|
|
|
self._COLLECT_PAGE, sort_field, sort_asc,
|
|
|
)
|
|
|
qs = raw.get('quotes', []) or []
|
|
|
if yahoo_total is None:
|
|
|
yahoo_total = raw.get('total') or raw.get('count') or 0
|
|
|
for q in qs:
|
|
|
sym = q.get('symbol')
|
|
|
if sym and sym not in seen:
|
|
|
seen[sym] = q
|
|
|
pages += 1
|
|
|
offset += self._COLLECT_PAGE
|
|
|
if not qs or len(qs) < self._COLLECT_PAGE:
|
|
|
break
|
|
|
if yahoo_total and offset >= yahoo_total:
|
|
|
break
|
|
|
truncated = bool(
|
|
|
pages >= self._COLLECT_MAX_PAGES
|
|
|
and yahoo_total
|
|
|
and offset < yahoo_total
|
|
|
)
|
|
|
return list(seen.values()), (yahoo_total or len(seen)), truncated
|
|
|
|
|
|
async def screen_preset(self, preset: str, page: int = 1, page_size: int = 25) -> dict:
|
|
|
"""Fetch a Yahoo Finance predefined screener (e.g. day_gainers)."""
|
|
|
start_time = time.time()
|
|
|
page_size = max(1, min(page_size, 250))
|
|
|
page = max(1, page)
|
|
|
offset = (page - 1) * page_size
|
|
|
|
|
|
loop = asyncio.get_event_loop()
|
|
|
raw = await loop.run_in_executor(
|
|
|
None, self._screen_preset_sync, preset, offset, page_size
|
|
|
)
|
|
|
|
|
|
quotes = raw.get('quotes', [])
|
|
|
# Yahoo returns the full match count under 'total'; 'count' is only the
|
|
|
# number of rows in THIS page (== size). Prefer 'total' so total_pages
|
|
|
# reflects the real result set, not a single page.
|
|
|
total_available = raw.get('total') or raw.get('count') or len(quotes)
|
|
|
total_pages = max(1, (total_available + page_size - 1) // page_size)
|
|
|
|
|
|
return {
|
|
|
'stocks': [self._parse_quote(q) for q in quotes],
|
|
|
'total_available': total_available,
|
|
|
'returned_count': len(quotes),
|
|
|
'page': page,
|
|
|
'page_size': page_size,
|
|
|
'total_pages': total_pages,
|
|
|
'query_time_seconds': round(time.time() - start_time, 3),
|
|
|
'metadata': {
|
|
|
'preset': preset,
|
|
|
'source': 'yfinance_screen_preset',
|
|
|
},
|
|
|
}
|
|
|
|
|
|
async def screen_stocks(
|
|
|
self,
|
|
|
market_cap_min: Optional[float] = None,
|
|
|
market_cap_max: Optional[float] = None,
|
|
|
exchange: Optional[str] = None,
|
|
|
min_avg_volume: Optional[int] = None,
|
|
|
min_dollar_volume: Optional[float] = None,
|
|
|
exclude_types: Optional[str] = None,
|
|
|
sector: Optional[str] = None,
|
|
|
pe_min: Optional[float] = None,
|
|
|
pe_max: Optional[float] = None,
|
|
|
price_min: Optional[float] = None,
|
|
|
price_max: Optional[float] = None,
|
|
|
page: int = 1,
|
|
|
page_size: int = 100,
|
|
|
sort_by: str = "market_cap",
|
|
|
sort_ascending: bool = False,
|
|
|
) -> dict:
|
|
|
"""Screen stocks with the given filters and return paginated results.
|
|
|
|
|
|
``min_dollar_volume`` is opt-in. When it is None the behaviour is
|
|
|
byte-for-byte identical to before this parameter existed (single Yahoo
|
|
|
page, Yahoo applies ``min_avg_volume``). When set, the share-count
|
|
|
``min_avg_volume`` filter is intentionally NOT pushed to Yahoo — that
|
|
|
would drop high-priced, low-share-volume names (BLK, KLAC, …) at the
|
|
|
source before dollar volume can be evaluated — and the complete match
|
|
|
set is fetched and post-filtered on price × averageDailyVolume3Month.
|
|
|
"""
|
|
|
start_time = time.time()
|
|
|
|
|
|
page_size = max(1, min(page_size, 250))
|
|
|
page = max(1, page)
|
|
|
|
|
|
dollar_mode = min_dollar_volume is not None
|
|
|
# In dollar-volume mode the share-count avg-volume gate is replaced by
|
|
|
# the dollar-volume gate, so it must not be sent to Yahoo.
|
|
|
effective_min_avg_volume = None if dollar_mode else min_avg_volume
|
|
|
|
|
|
query = self._build_query(
|
|
|
market_cap_min=market_cap_min,
|
|
|
market_cap_max=market_cap_max,
|
|
|
exchange=exchange,
|
|
|
min_avg_volume=effective_min_avg_volume,
|
|
|
sector=sector,
|
|
|
pe_min=pe_min,
|
|
|
pe_max=pe_max,
|
|
|
price_min=price_min,
|
|
|
price_max=price_max,
|
|
|
)
|
|
|
|
|
|
sort_field = self.SORT_FIELD_MAP.get(sort_by, "intradaymarketcap")
|
|
|
|
|
|
# Post-filter: remove non-equity types if requested
|
|
|
exclude_type_set = set()
|
|
|
if exclude_types:
|
|
|
exclude_type_set = {t.strip().upper() for t in exclude_types.split(',')}
|
|
|
|
|
|
extra_meta: dict = {}
|
|
|
|
|
|
if not dollar_mode:
|
|
|
# ---- Unchanged legacy path (zero regression when opt-in is off) ----
|
|
|
offset = (page - 1) * page_size
|
|
|
loop = asyncio.get_event_loop()
|
|
|
raw = await loop.run_in_executor(
|
|
|
None,
|
|
|
self._screen_sync,
|
|
|
query,
|
|
|
offset,
|
|
|
page_size,
|
|
|
sort_field,
|
|
|
sort_ascending,
|
|
|
)
|
|
|
|
|
|
quotes = raw.get('quotes', [])
|
|
|
# Yahoo's 'total' is the full match count; 'count' is only this
|
|
|
# page's row count. Prefer 'total' so total_pages is correct and
|
|
|
# clients that paginate by total_pages don't stop after page 1.
|
|
|
total_available = raw.get('total') or raw.get('count') or len(quotes)
|
|
|
|
|
|
stocks = []
|
|
|
for quote in quotes:
|
|
|
if exclude_type_set:
|
|
|
qt = (quote.get('quoteType') or '').upper()
|
|
|
if qt in exclude_type_set:
|
|
|
continue
|
|
|
stocks.append(self._parse_quote(quote))
|
|
|
else:
|
|
|
# ---- Opt-in dollar-volume path: fetch full set, post-filter ----
|
|
|
all_quotes, _yahoo_total, truncated = await self._collect_all_quotes(
|
|
|
query, sort_field, sort_ascending
|
|
|
)
|
|
|
|
|
|
filtered = []
|
|
|
for quote in all_quotes:
|
|
|
if exclude_type_set:
|
|
|
qt = (quote.get('quoteType') or '').upper()
|
|
|
if qt in exclude_type_set:
|
|
|
continue
|
|
|
price = quote.get('regularMarketPrice')
|
|
|
avg_vol = quote.get('averageDailyVolume3Month')
|
|
|
if price is None or avg_vol is None:
|
|
|
continue
|
|
|
if price * avg_vol < min_dollar_volume:
|
|
|
continue
|
|
|
filtered.append(self._parse_quote(quote))
|
|
|
|
|
|
sort_key = self.POST_SORT_KEY.get(sort_by, "market_cap")
|
|
|
# Partition so rows missing the sort field are always last,
|
|
|
# regardless of sort direction, and so str/num keys never mix.
|
|
|
present = [s for s in filtered if s.get(sort_key) is not None]
|
|
|
missing = [s for s in filtered if s.get(sort_key) is None]
|
|
|
present.sort(key=lambda s: s.get(sort_key), reverse=not sort_ascending)
|
|
|
filtered = present + missing
|
|
|
|
|
|
total_available = len(filtered)
|
|
|
start = (page - 1) * page_size
|
|
|
stocks = filtered[start:start + page_size]
|
|
|
|
|
|
extra_meta['dollar_volume_mode'] = True
|
|
|
extra_meta['fetched_universe'] = len(all_quotes)
|
|
|
if min_avg_volume is not None:
|
|
|
extra_meta['note_min_avg_volume'] = (
|
|
|
'min_avg_volume ignored because min_dollar_volume is set '
|
|
|
'(dollar volume replaces the share-count liquidity gate)'
|
|
|
)
|
|
|
if truncated:
|
|
|
extra_meta['truncated'] = True
|
|
|
extra_meta['note_truncated'] = (
|
|
|
'Yahoo result set exceeded internal fetch cap; widen '
|
|
|
'market_cap_min to narrow the universe for completeness'
|
|
|
)
|
|
|
|
|
|
query_time = time.time() - start_time
|
|
|
total_pages = max(1, (total_available + page_size - 1) // page_size)
|
|
|
|
|
|
filters_applied = {}
|
|
|
if market_cap_min is not None:
|
|
|
filters_applied['market_cap_min'] = market_cap_min
|
|
|
if market_cap_max is not None:
|
|
|
filters_applied['market_cap_max'] = market_cap_max
|
|
|
if exchange:
|
|
|
filters_applied['exchange'] = exchange
|
|
|
if min_avg_volume is not None and not dollar_mode:
|
|
|
filters_applied['min_avg_volume'] = min_avg_volume
|
|
|
if min_dollar_volume is not None:
|
|
|
filters_applied['min_dollar_volume'] = min_dollar_volume
|
|
|
if exclude_types:
|
|
|
filters_applied['exclude_types'] = exclude_types
|
|
|
if sector:
|
|
|
filters_applied['sector'] = sector
|
|
|
if pe_min is not None:
|
|
|
filters_applied['pe_min'] = pe_min
|
|
|
if pe_max is not None:
|
|
|
filters_applied['pe_max'] = pe_max
|
|
|
if price_min is not None:
|
|
|
filters_applied['price_min'] = price_min
|
|
|
if price_max is not None:
|
|
|
filters_applied['price_max'] = price_max
|
|
|
|
|
|
return {
|
|
|
'stocks': stocks,
|
|
|
'total_available': total_available,
|
|
|
'returned_count': len(stocks),
|
|
|
'page': page,
|
|
|
'page_size': page_size,
|
|
|
'total_pages': total_pages,
|
|
|
'query_time_seconds': round(query_time, 3),
|
|
|
'metadata': {
|
|
|
'filters_applied': filters_applied,
|
|
|
'sort_by': sort_by,
|
|
|
'sort_ascending': sort_ascending,
|
|
|
'source': 'yfinance_screen',
|
|
|
'note': 'sector/industry not included in per-stock response (Yahoo API limitation)',
|
|
|
**extra_meta,
|
|
|
},
|
|
|
}
|
|
|
|
|
|
|
|
|
screener_service = ScreenerService()
|