From d513f6492223a22484668cd7f77011d44562a0d5 Mon Sep 17 00:00:00 2001 From: I Luk Kim Date: Sun, 17 May 2026 17:19:20 -0700 Subject: [PATCH] =?UTF-8?q?fix:=20screener=20total=5Favailable=20=ED=95=84?= =?UTF-8?q?=EB=93=9C=20=EC=98=A4=EB=8F=85=20=EC=88=98=EC=A0=95=20+=20feat:?= =?UTF-8?q?=20min=5Fdollar=5Fvolume=20opt-in=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [사안 A] total_available 정확성 수정 - raw.get('count')(페이지당 행수=page_size)를 raw.get('total')(실제 매칭 총수)보다 먼저 읽어 total_available=250/total_pages=1로 잘못 보고됨 - 'total' 우선으로 교정 (screen_stocks + screen_preset) - 732행 집합에서 total_available 250→732, total_pages 1→3 검증 [사안 B] min_dollar_volume opt-in 파라미터 신규 - 가격(regularMarketPrice) × averageDailyVolume3Month 기준 달러거래량 필터 - 기본 OFF: 미지정 시 기존 경로 byte 동일 (회귀 0 검증) - 지정 시 min_avg_volume을 Yahoo에 전달하지 않음 (고가·저주식수 우량주 BLK/KLAC가 소스에서 잘리는 것 방지) → 달러거래량 게이트가 대체 - _collect_all_quotes로 전체 매칭 집합 페이지네이션(24p/6000행 상한, 초과 시 truncated 플래그) 후 post-filter + 서버측 재정렬 - @with_cache key_params에 min_dollar_volume 등록 (미지정/지정 캐시 분리) - 검증: min_dollar_volume=1e8 시 BLK/KLAC(p1)·URI/GRMN(p2) 전부 포함, exclude_types·market_cap_min·price_min 동시 정상 적용 Co-Authored-By: Claude Opus 4.7 --- app/api/v1/endpoints/screener.py | 17 ++- app/services/screener_service.py | 182 ++++++++++++++++++++++++++----- 2 files changed, 171 insertions(+), 28 deletions(-) diff --git a/app/api/v1/endpoints/screener.py b/app/api/v1/endpoints/screener.py index f70a4df..ce4c301 100644 --- a/app/api/v1/endpoints/screener.py +++ b/app/api/v1/endpoints/screener.py @@ -22,7 +22,8 @@ logger = logging.getLogger("app.api.v1.screener") ttl=300, key_params=[ "market_cap_min", "market_cap_max", "exchange", "min_avg_volume", - "exclude_types", "sector", "pe_min", "pe_max", "price_min", "price_max", + "min_dollar_volume", "exclude_types", "sector", "pe_min", "pe_max", + "price_min", "price_max", "page", "page_size", "sort_by", "sort_ascending", ], ) @@ -40,7 +41,18 @@ async def screen_stocks( "Omit for all US exchanges.", ), min_avg_volume: Optional[int] = Query( - None, ge=0, description="Minimum 3-month average daily volume (e.g. 500000)" + None, ge=0, description="Minimum 3-month average daily volume in SHARES (e.g. 500000)" + ), + min_dollar_volume: Optional[float] = Query( + None, + ge=0, + description="Opt-in. Minimum 3-month average daily DOLLAR volume " + "(regularMarketPrice × averageDailyVolume3Month), e.g. " + "100000000 for $100M/day. When set, the full match set is " + "fetched and post-filtered, and the share-count " + "min_avg_volume gate is NOT applied (it would exclude " + "high-priced low-share-volume names like BLK/KLAC at the " + "source). When omitted, screener behaviour is unchanged.", ), exclude_types: Optional[str] = Query( None, @@ -99,6 +111,7 @@ async def screen_stocks( market_cap_max=market_cap_max, exchange=exchange, min_avg_volume=min_avg_volume, + min_dollar_volume=min_dollar_volume, exclude_types=exclude_types, sector=sector, pe_min=pe_min, diff --git a/app/services/screener_service.py b/app/services/screener_service.py index b4d53e0..04971cd 100644 --- a/app/services/screener_service.py +++ b/app/services/screener_service.py @@ -46,6 +46,23 @@ class ScreenerService: "price_to_book": "pricebook", } + # Maps API sort_by names to keys on the parsed-stock dict. Used only in + # min_dollar_volume mode, where results are assembled across multiple + # Yahoo pages and must be re-sorted server-side. + POST_SORT_KEY = { + "market_cap": "market_cap", + "volume": "volume", + "avg_volume": "avg_volume_3m", + "price": "price", + "pe_ratio": "pe_ratio", + "change_percent": "change_percent", + "name": "name", + "eps": "eps_ttm", + "dividend_yield": "dividend_yield", + "forward_pe": "forward_pe", + "price_to_book": "price_to_book", + } + def _build_query( self, market_cap_min: Optional[float], @@ -160,6 +177,47 @@ class ScreenerService: import yfinance_plus as yf return yf.screen(preset, offset=offset, count=count) + # Yahoo's screen() max page is 250; cap total internal fetch depth so a + # broad query (e.g. low market_cap_min) can't issue unbounded requests. + _COLLECT_PAGE = 250 + _COLLECT_MAX_PAGES = 24 # up to 6000 rows + + async def _collect_all_quotes(self, query, sort_field: str, sort_asc: bool): + """Page through Yahoo screen() until the full match set is collected. + + Returns (quotes, yahoo_total, truncated). Used only by the + min_dollar_volume path, which must post-filter the complete set. + """ + loop = asyncio.get_event_loop() + seen: dict = {} + offset = 0 + yahoo_total = None + pages = 0 + while pages < self._COLLECT_MAX_PAGES: + raw = await loop.run_in_executor( + None, self._screen_sync, query, offset, + self._COLLECT_PAGE, sort_field, sort_asc, + ) + qs = raw.get('quotes', []) or [] + if yahoo_total is None: + yahoo_total = raw.get('total') or raw.get('count') or 0 + for q in qs: + sym = q.get('symbol') + if sym and sym not in seen: + seen[sym] = q + pages += 1 + offset += self._COLLECT_PAGE + if not qs or len(qs) < self._COLLECT_PAGE: + break + if yahoo_total and offset >= yahoo_total: + break + truncated = bool( + pages >= self._COLLECT_MAX_PAGES + and yahoo_total + and offset < yahoo_total + ) + return list(seen.values()), (yahoo_total or len(seen)), truncated + async def screen_preset(self, preset: str, page: int = 1, page_size: int = 25) -> dict: """Fetch a Yahoo Finance predefined screener (e.g. day_gainers).""" start_time = time.time() @@ -173,7 +231,10 @@ class ScreenerService: ) quotes = raw.get('quotes', []) - total_available = raw.get('count') or raw.get('total') or len(quotes) + # Yahoo returns the full match count under 'total'; 'count' is only the + # number of rows in THIS page (== size). Prefer 'total' so total_pages + # reflects the real result set, not a single page. + total_available = raw.get('total') or raw.get('count') or len(quotes) total_pages = max(1, (total_available + page_size - 1) // page_size) return { @@ -196,6 +257,7 @@ class ScreenerService: market_cap_max: Optional[float] = None, exchange: Optional[str] = None, min_avg_volume: Optional[int] = None, + min_dollar_volume: Optional[float] = None, exclude_types: Optional[str] = None, sector: Optional[str] = None, pe_min: Optional[float] = None, @@ -207,17 +269,31 @@ class ScreenerService: sort_by: str = "market_cap", sort_ascending: bool = False, ) -> dict: - """Screen stocks with the given filters and return paginated results.""" + """Screen stocks with the given filters and return paginated results. + + ``min_dollar_volume`` is opt-in. When it is None the behaviour is + byte-for-byte identical to before this parameter existed (single Yahoo + page, Yahoo applies ``min_avg_volume``). When set, the share-count + ``min_avg_volume`` filter is intentionally NOT pushed to Yahoo — that + would drop high-priced, low-share-volume names (BLK, KLAC, …) at the + source before dollar volume can be evaluated — and the complete match + set is fetched and post-filtered on price × averageDailyVolume3Month. + """ start_time = time.time() page_size = max(1, min(page_size, 250)) page = max(1, page) + dollar_mode = min_dollar_volume is not None + # In dollar-volume mode the share-count avg-volume gate is replaced by + # the dollar-volume gate, so it must not be sent to Yahoo. + effective_min_avg_volume = None if dollar_mode else min_avg_volume + query = self._build_query( market_cap_min=market_cap_min, market_cap_max=market_cap_max, exchange=exchange, - min_avg_volume=min_avg_volume, + min_avg_volume=effective_min_avg_volume, sector=sector, pe_min=pe_min, pe_max=pe_max, @@ -226,35 +302,86 @@ class ScreenerService: ) sort_field = self.SORT_FIELD_MAP.get(sort_by, "intradaymarketcap") - offset = (page - 1) * page_size - - loop = asyncio.get_event_loop() - raw = await loop.run_in_executor( - None, - self._screen_sync, - query, - offset, - page_size, - sort_field, - sort_ascending, - ) - - quotes = raw.get('quotes', []) - # yfinance may return total count under 'count' or 'total' - total_available = raw.get('count') or raw.get('total') or len(quotes) # Post-filter: remove non-equity types if requested exclude_type_set = set() if exclude_types: exclude_type_set = {t.strip().upper() for t in exclude_types.split(',')} - stocks = [] - for quote in quotes: - if exclude_type_set: - qt = (quote.get('quoteType') or '').upper() - if qt in exclude_type_set: + extra_meta: dict = {} + + if not dollar_mode: + # ---- Unchanged legacy path (zero regression when opt-in is off) ---- + offset = (page - 1) * page_size + loop = asyncio.get_event_loop() + raw = await loop.run_in_executor( + None, + self._screen_sync, + query, + offset, + page_size, + sort_field, + sort_ascending, + ) + + quotes = raw.get('quotes', []) + # Yahoo's 'total' is the full match count; 'count' is only this + # page's row count. Prefer 'total' so total_pages is correct and + # clients that paginate by total_pages don't stop after page 1. + total_available = raw.get('total') or raw.get('count') or len(quotes) + + stocks = [] + for quote in quotes: + if exclude_type_set: + qt = (quote.get('quoteType') or '').upper() + if qt in exclude_type_set: + continue + stocks.append(self._parse_quote(quote)) + else: + # ---- Opt-in dollar-volume path: fetch full set, post-filter ---- + all_quotes, _yahoo_total, truncated = await self._collect_all_quotes( + query, sort_field, sort_ascending + ) + + filtered = [] + for quote in all_quotes: + if exclude_type_set: + qt = (quote.get('quoteType') or '').upper() + if qt in exclude_type_set: + continue + price = quote.get('regularMarketPrice') + avg_vol = quote.get('averageDailyVolume3Month') + if price is None or avg_vol is None: continue - stocks.append(self._parse_quote(quote)) + if price * avg_vol < min_dollar_volume: + continue + filtered.append(self._parse_quote(quote)) + + sort_key = self.POST_SORT_KEY.get(sort_by, "market_cap") + # Partition so rows missing the sort field are always last, + # regardless of sort direction, and so str/num keys never mix. + present = [s for s in filtered if s.get(sort_key) is not None] + missing = [s for s in filtered if s.get(sort_key) is None] + present.sort(key=lambda s: s.get(sort_key), reverse=not sort_ascending) + filtered = present + missing + + total_available = len(filtered) + start = (page - 1) * page_size + stocks = filtered[start:start + page_size] + + extra_meta['dollar_volume_mode'] = True + extra_meta['fetched_universe'] = len(all_quotes) + if min_avg_volume is not None: + extra_meta['note_min_avg_volume'] = ( + 'min_avg_volume ignored because min_dollar_volume is set ' + '(dollar volume replaces the share-count liquidity gate)' + ) + if truncated: + extra_meta['truncated'] = True + extra_meta['note_truncated'] = ( + 'Yahoo result set exceeded internal fetch cap; widen ' + 'market_cap_min to narrow the universe for completeness' + ) query_time = time.time() - start_time total_pages = max(1, (total_available + page_size - 1) // page_size) @@ -266,8 +393,10 @@ class ScreenerService: filters_applied['market_cap_max'] = market_cap_max if exchange: filters_applied['exchange'] = exchange - if min_avg_volume is not None: + if min_avg_volume is not None and not dollar_mode: filters_applied['min_avg_volume'] = min_avg_volume + if min_dollar_volume is not None: + filters_applied['min_dollar_volume'] = min_dollar_volume if exclude_types: filters_applied['exclude_types'] = exclude_types if sector: @@ -295,6 +424,7 @@ class ScreenerService: 'sort_ascending': sort_ascending, 'source': 'yfinance_screen', 'note': 'sector/industry not included in per-stock response (Yahoo API limitation)', + **extra_meta, }, }