diff --git a/app/api/v1/endpoints/stocks.py b/app/api/v1/endpoints/stocks.py index 95b2b58..7f940a0 100644 --- a/app/api/v1/endpoints/stocks.py +++ b/app/api/v1/endpoints/stocks.py @@ -301,26 +301,49 @@ async def get_52week_gainers( @router.get( "/gainers", - summary="Today's top gaining stocks (Yahoo Finance day_gainers preset)", + summary="Today's top gaining stocks (Yahoo Finance day_gainers)", ) async def get_day_gainers( - page: int = Query(1, ge=1, description="Page number (1-based)"), - page_size: int = Query(25, ge=1, le=250, description="Results per page (max 250)"), + count: int = Query(100, ge=1, le=250, description="Number of results to return (max 250)"), ): """ - Top gaining stocks for today using Yahoo Finance's `day_gainers` preset. + Top gaining stocks for today via Yahoo Finance's `day_gainers` predefined screener. Criteria: price change > 3%, market cap >= $2B, price >= $5, volume > 15,000. Sorted by percent change descending. Real-time — no cache. + + Data source: `query1.finance.yahoo.com/v1/finance/screener/predefined/saved` + with browser fingerprint rotation for 429 bypass. """ - from app.services.screener_service import screener_service + import asyncio + import time as _time + from app.services.gainers.yahoo_client import fetch_day_gainers_sync, _parse_quote + + start_ts = _time.time() + loop = asyncio.get_event_loop() try: - return await screener_service.screen_preset("day_gainers", page=page, page_size=page_size) - except RuntimeError as e: - raise HTTPException(status_code=503, detail=str(e)) + raw_quotes, total, error = await loop.run_in_executor( + None, fetch_day_gainers_sync, count, 0 + ) except Exception as e: - logger.error("Day gainers error: %s", e, exc_info=True) - raise HTTPException(status_code=500, detail=f"Screener query failed: {str(e)}") + logger.error("Day gainers fetch error: %s", e, exc_info=True) + raise HTTPException(status_code=503, detail=str(e)) + + if error: + raise HTTPException(status_code=503, detail=f"Yahoo Finance error: {error}") + + stocks = [_parse_quote(q) for q in raw_quotes] + return { + "stocks": stocks, + "total_available": total, + "returned_count": len(stocks), + "query_time_seconds": round(_time.time() - start_ts, 3), + "metadata": { + "source": "yahoo_finance_predefined_screener", + "preset": "day_gainers", + "criteria": "change>3%, mktcap>=$2B, price>=$5, volume>15k", + }, + } @router.get( diff --git a/app/services/gainers/collector.py b/app/services/gainers/collector.py index e8b3c96..fa7b9a4 100644 --- a/app/services/gainers/collector.py +++ b/app/services/gainers/collector.py @@ -1,6 +1,8 @@ """ -Gainer snapshot collector — fetches top 100 day_gainers and stores per 5-min slot. +Gainer snapshot collector — fetches top 200 day_gainers every 5-min slot. +Uses direct Yahoo Finance API via yahoo_client (no yfinance_plus). """ +import asyncio import logging from datetime import datetime, timezone @@ -8,23 +10,28 @@ logger = logging.getLogger(__name__) def _floor_to_5min(dt: datetime) -> datetime: - """Round down to the nearest 5-minute boundary.""" return dt.replace(minute=dt.minute - (dt.minute % 5), second=0, microsecond=0) async def collect_gainer_snapshot() -> int: - """Fetch 100 day_gainers and bulk-insert into gainer_snapshots. Returns inserted count.""" - from app.services.screener_service import screener_service + """Fetch 200 day_gainers and bulk-insert into gainer_snapshots. Returns inserted count.""" + from app.services.gainers.yahoo_client import fetch_day_gainers_sync, _parse_quote from app.core.database import AsyncSessionLocal from app.models.gainer_snapshot import GainerSnapshot from sqlalchemy.dialects.postgresql import insert as pg_insert snapshot_at = _floor_to_5min(datetime.now(timezone.utc)) - result = await screener_service.screen_preset("day_gainers", page=1, page_size=100) - stocks = result.get("stocks", []) - if not stocks: - logger.warning("[Gainers] No stocks returned from day_gainers preset") + loop = asyncio.get_event_loop() + raw_quotes, total, error = await loop.run_in_executor( + None, fetch_day_gainers_sync, 200, 0 + ) + + if error: + logger.error("[Gainers] fetch failed: %s", error) + return 0 + if not raw_quotes: + logger.warning("[Gainers] No stocks returned (total=%d)", total) return 0 rows = [ @@ -46,7 +53,7 @@ async def collect_gainer_snapshot() -> int: "fifty_two_week_high": s.get("fifty_two_week_high"), "fifty_two_week_low": s.get("fifty_two_week_low"), } - for rank, s in enumerate(stocks, start=1) + for rank, s in enumerate((_parse_quote(q) for q in raw_quotes), start=1) ] async with AsyncSessionLocal() as db: @@ -55,5 +62,5 @@ async def collect_gainer_snapshot() -> int: await db.execute(stmt) await db.commit() - logger.info("[Gainers] snapshot %s — %d rows", snapshot_at.isoformat(), len(rows)) + logger.info("[Gainers] snapshot %s — %d/%d rows", snapshot_at.isoformat(), len(rows), total) return len(rows) diff --git a/app/services/gainers/yahoo_client.py b/app/services/gainers/yahoo_client.py new file mode 100644 index 0000000..c84e8d9 --- /dev/null +++ b/app/services/gainers/yahoo_client.py @@ -0,0 +1,206 @@ +""" +Direct Yahoo Finance predefined screener HTTP client. + +Uses curl_cffi with browser fingerprint rotation (round-robin session pool) +to avoid 429 rate limiting — no yfinance_plus dependency. + +Endpoint: https://query1.finance.yahoo.com/v1/finance/screener/predefined/saved +""" +import logging +import random +import threading +import time +from typing import Optional + +logger = logging.getLogger(__name__) + +_BASE_URL = "https://query1.finance.yahoo.com/v1/finance/screener/predefined/saved" + +_BROWSER_PROFILES = [ + { + "impersonate": "chrome120", + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "sec_ch_ua": '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"', + "sec_ch_ua_platform": '"macOS"', + }, + { + "impersonate": "chrome110", + "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36", + "sec_ch_ua": '"Not A;Brand";v="8", "Chromium";v="110", "Google Chrome";v="110"', + "sec_ch_ua_platform": '"Windows"', + }, + { + "impersonate": "edge99", + "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36 Edg/99.0.1150.36", + "sec_ch_ua": '"Not A;Brand";v="99", "Chromium";v="99", "Microsoft Edge";v="99"', + "sec_ch_ua_platform": '"Windows"', + }, + { + "impersonate": "safari15_5", + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15", + "sec_ch_ua": None, + "sec_ch_ua_platform": None, + }, +] + +_ACCEPT_LANGUAGES = [ + "en-US,en;q=0.9", + "en-US,en;q=0.9,ko;q=0.8", + "en-GB,en;q=0.9", +] + +# Exchange code → friendly name (same mapping as screener_service) +_EXCHANGE_MAP = { + "NYQ": "NYSE", "NMS": "NASDAQ", "NGM": "NASDAQ", + "NCM": "NASDAQ", "ASE": "AMEX", "PCX": "NYSE_ARCA", +} + + +def _parse_quote(q: dict) -> dict: + """Normalize a raw Yahoo Finance quote dict to our standard format.""" + code = q.get("exchange", "") + volume = q.get("regularMarketVolume") + avg_vol = q.get("averageDailyVolume3Month") + return { + "symbol": q.get("symbol", ""), + "name": q.get("shortName") or q.get("longName"), + "exchange": _EXCHANGE_MAP.get(code, code), + "exchange_code": code, + "quote_type": q.get("quoteType"), + "market_cap": q.get("marketCap"), + "price": q.get("regularMarketPrice"), + "change_percent": q.get("regularMarketChangePercent"), + "volume": int(volume) if volume is not None else None, + "avg_volume_3m": int(avg_vol) if avg_vol is not None else None, + "shares_outstanding": q.get("sharesOutstanding"), + "pe_ratio": q.get("trailingPE"), + "forward_pe": q.get("forwardPE"), + "eps_ttm": q.get("epsTrailingTwelveMonths"), + "dividend_yield": q.get("trailingAnnualDividendYield"), + "fifty_two_week_high": q.get("fiftyTwoWeekHigh"), + "fifty_two_week_low": q.get("fiftyTwoWeekLow"), + "book_value": q.get("bookValue"), + "price_to_book": q.get("priceToBook"), + "analyst_rating": q.get("averageAnalystRating"), + } + + +class _YahooScreenerClient: + _POOL_SIZE = 4 + _MAX_RETRIES = 4 + + def __init__(self): + self._lock = threading.Lock() + self._idx = 0 + self._sessions: list = [] + self._initialized = False + + def _ensure_init(self): + if self._initialized: + return + with self._lock: + if self._initialized: + return + profiles = random.sample(_BROWSER_PROFILES, min(self._POOL_SIZE, len(_BROWSER_PROFILES))) + self._sessions = [self._make_session(p) for p in profiles] + self._initialized = True + + def _make_session(self, profile: dict): + from curl_cffi import requests as cr + try: + session = cr.Session(impersonate=profile["impersonate"]) + except Exception: + session = cr.Session(impersonate="chrome") + + headers = { + "User-Agent": profile["user_agent"], + "Accept": "application/json, text/plain, */*", + "Accept-Language": random.choice(_ACCEPT_LANGUAGES), + "Accept-Encoding": "gzip, deflate, br", + "Cache-Control": "no-cache", + "Origin": "https://finance.yahoo.com", + "Referer": "https://finance.yahoo.com/screener/predefined/day_gainers/", + } + if profile.get("sec_ch_ua"): + headers["Sec-Ch-Ua"] = profile["sec_ch_ua"] + headers["Sec-Ch-Ua-Mobile"] = "?0" + headers["Sec-Ch-Ua-Platform"] = profile["sec_ch_ua_platform"] + headers["Sec-Fetch-Dest"] = "empty" + headers["Sec-Fetch-Mode"] = "cors" + headers["Sec-Fetch-Site"] = "same-site" + session.headers.update(headers) + return session + + def _next_session(self): + with self._lock: + s = self._sessions[self._idx % len(self._sessions)] + self._idx += 1 + return s + + def _rotate(self): + """Replace the next slot with a fresh session on a random profile.""" + profile = random.choice(_BROWSER_PROFILES) + new_s = self._make_session(profile) + with self._lock: + self._sessions[self._idx % len(self._sessions)] = new_s + + def fetch_sync( + self, + preset: str = "day_gainers", + count: int = 200, + start: int = 0, + ) -> tuple[list, int, Optional[str]]: + """ + Synchronous fetch — run via asyncio.run_in_executor. + + Returns (quotes: list[dict], total: int, error: str|None). + """ + self._ensure_init() + params = { + "formatted": "false", + "lang": "en-US", + "region": "US", + "scrIds": preset, + "count": count, + "start": start, + } + last_exc: Exception = RuntimeError("no attempts made") + for attempt in range(self._MAX_RETRIES): + if attempt > 0: + delay = 2 ** attempt + random.uniform(0, 1) + logger.info("[YahooClient] backoff %.1fs (attempt %d)", delay, attempt) + time.sleep(delay) + self._rotate() + + session = self._next_session() + try: + resp = session.get(_BASE_URL, params=params, timeout=15) + if resp.status_code in (429, 401): + logger.warning("[YahooClient] HTTP %d — rotating session", resp.status_code) + self._rotate() + last_exc = RuntimeError(f"HTTP {resp.status_code}") + continue + resp.raise_for_status() + result = (resp.json().get("finance", {}).get("result") or [{}])[0] + quotes = result.get("quotes", []) + total = result.get("total", len(quotes)) + return quotes, total, None + except Exception as e: + last_exc = e + if any(x in str(e) for x in ("429", "401", "rate")): + self._rotate() + logger.warning("[YahooClient] attempt %d: %s", attempt, e) + + return [], 0, str(last_exc) + + +# Module-level singleton — lazy-initialized on first call +_client = _YahooScreenerClient() + + +def fetch_day_gainers_sync(count: int = 200, start: int = 0) -> tuple[list, int, Optional[str]]: + """ + Fetch day_gainers from Yahoo Finance. Returns (raw_quotes, total, error). + Call via asyncio.run_in_executor — blocking I/O. + """ + return _client.fetch_sync("day_gainers", count=count, start=start)