stock-oracle/app/services/gainers/yahoo_client.py

"""
Direct Yahoo Finance predefined screener HTTP client.

Uses curl_cffi with browser fingerprint rotation (round-robin session pool)
to avoid 429 rate limiting — no yfinance_plus dependency.

Endpoint: https://query1.finance.yahoo.com/v1/finance/screener/predefined/saved
"""
import logging
import random
import threading
import time
from typing import Optional

logger = logging.getLogger(__name__)

_BASE_URL = "https://query1.finance.yahoo.com/v1/finance/screener/predefined/saved"

_BROWSER_PROFILES = [
    {
        "impersonate": "chrome120",
        "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
        "sec_ch_ua": '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
        "sec_ch_ua_platform": '"macOS"',
    },
    {
        "impersonate": "chrome110",
        "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36",
        "sec_ch_ua": '"Not A;Brand";v="8", "Chromium";v="110", "Google Chrome";v="110"',
        "sec_ch_ua_platform": '"Windows"',
    },
    {
        "impersonate": "edge99",
        "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36 Edg/99.0.1150.36",
        "sec_ch_ua": '"Not A;Brand";v="99", "Chromium";v="99", "Microsoft Edge";v="99"',
        "sec_ch_ua_platform": '"Windows"',
    },
    {
        "impersonate": "safari15_5",
        "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15",
        "sec_ch_ua": None,
        "sec_ch_ua_platform": None,
    },
]

_ACCEPT_LANGUAGES = [
    "en-US,en;q=0.9",
    "en-US,en;q=0.9,ko;q=0.8",
    "en-GB,en;q=0.9",
]

# Exchange code → friendly name (same mapping as screener_service)
_EXCHANGE_MAP = {
    "NYQ": "NYSE", "NMS": "NASDAQ", "NGM": "NASDAQ",
    "NCM": "NASDAQ", "ASE": "AMEX", "PCX": "NYSE_ARCA",
}


def _parse_quote(q: dict) -> dict:
    """Normalize a raw Yahoo Finance quote dict to our standard format."""
    code = q.get("exchange", "")
    volume = q.get("regularMarketVolume")
    avg_vol = q.get("averageDailyVolume3Month")
    return {
        "symbol": q.get("symbol", ""),
        "name": q.get("shortName") or q.get("longName"),
        "exchange": _EXCHANGE_MAP.get(code, code),
        "exchange_code": code,
        "quote_type": q.get("quoteType"),
        "market_cap": q.get("marketCap"),
        "price": q.get("regularMarketPrice"),
        "change_percent": q.get("regularMarketChangePercent"),
        "volume": int(volume) if volume is not None else None,
        "avg_volume_3m": int(avg_vol) if avg_vol is not None else None,
        "shares_outstanding": q.get("sharesOutstanding"),
        "pe_ratio": q.get("trailingPE"),
        "forward_pe": q.get("forwardPE"),
        "eps_ttm": q.get("epsTrailingTwelveMonths"),
        "dividend_yield": q.get("trailingAnnualDividendYield"),
        "fifty_two_week_high": q.get("fiftyTwoWeekHigh"),
        "fifty_two_week_low": q.get("fiftyTwoWeekLow"),
        "book_value": q.get("bookValue"),
        "price_to_book": q.get("priceToBook"),
        "analyst_rating": q.get("averageAnalystRating"),
    }


class _YahooScreenerClient:
    _POOL_SIZE = 4
    _MAX_RETRIES = 4

    def __init__(self):
        self._lock = threading.Lock()
        self._idx = 0
        self._sessions: list = []
        self._initialized = False

    def _ensure_init(self):
        if self._initialized:
            return
        with self._lock:
            if self._initialized:
                return
            profiles = random.sample(_BROWSER_PROFILES, min(self._POOL_SIZE, len(_BROWSER_PROFILES)))
            self._sessions = [self._make_session(p) for p in profiles]
            self._initialized = True

    def _make_session(self, profile: dict):
        from curl_cffi import requests as cr
        try:
            session = cr.Session(impersonate=profile["impersonate"])
        except Exception:
            session = cr.Session(impersonate="chrome")

        headers = {
            "User-Agent": profile["user_agent"],
            "Accept": "application/json, text/plain, */*",
            "Accept-Language": random.choice(_ACCEPT_LANGUAGES),
            "Accept-Encoding": "gzip, deflate, br",
            "Cache-Control": "no-cache",
            "Origin": "https://finance.yahoo.com",
            "Referer": "https://finance.yahoo.com/screener/predefined/day_gainers/",
        }
        if profile.get("sec_ch_ua"):
            headers["Sec-Ch-Ua"] = profile["sec_ch_ua"]
            headers["Sec-Ch-Ua-Mobile"] = "?0"
            headers["Sec-Ch-Ua-Platform"] = profile["sec_ch_ua_platform"]
            headers["Sec-Fetch-Dest"] = "empty"
            headers["Sec-Fetch-Mode"] = "cors"
            headers["Sec-Fetch-Site"] = "same-site"
        session.headers.update(headers)
        return session

    def _next_session(self):
        with self._lock:
            s = self._sessions[self._idx % len(self._sessions)]
            self._idx += 1
            return s

    def _rotate(self):
        """Replace the next slot with a fresh session on a random profile."""
        profile = random.choice(_BROWSER_PROFILES)
        new_s = self._make_session(profile)
        with self._lock:
            self._sessions[self._idx % len(self._sessions)] = new_s

    def fetch_sync(
        self,
        preset: str = "day_gainers",
        count: int = 200,
        start: int = 0,
    ) -> tuple[list, int, Optional[str]]:
        """
        Synchronous fetch — run via asyncio.run_in_executor.

        Returns (quotes: list[dict], total: int, error: str|None).
        """
        self._ensure_init()
        params = {
            "formatted": "false",
            "lang": "en-US",
            "region": "US",
            "scrIds": preset,
            "count": count,
            "start": start,
        }
        last_exc: Exception = RuntimeError("no attempts made")
        for attempt in range(self._MAX_RETRIES):
            if attempt > 0:
                delay = 2 ** attempt + random.uniform(0, 1)
                logger.info("[YahooClient] backoff %.1fs (attempt %d)", delay, attempt)
                time.sleep(delay)
                self._rotate()

            session = self._next_session()
            try:
                resp = session.get(_BASE_URL, params=params, timeout=15)
                if resp.status_code in (429, 401):
                    logger.warning("[YahooClient] HTTP %d — rotating session", resp.status_code)
                    self._rotate()
                    last_exc = RuntimeError(f"HTTP {resp.status_code}")
                    continue
                resp.raise_for_status()
                result = (resp.json().get("finance", {}).get("result") or [{}])[0]
                quotes = result.get("quotes", [])
                total = result.get("total", len(quotes))
                return quotes, total, None
            except Exception as e:
                last_exc = e
                if any(x in str(e) for x in ("429", "401", "rate")):
                    self._rotate()
                logger.warning("[YahooClient] attempt %d: %s", attempt, e)

        return [], 0, str(last_exc)


# Module-level singleton — lazy-initialized on first call
_client = _YahooScreenerClient()


def fetch_day_gainers_sync(count: int = 200, start: int = 0) -> tuple[list, int, Optional[str]]:
    """
    Fetch day_gainers from Yahoo Finance. Returns (raw_quotes, total, error).
    Call via asyncio.run_in_executor — blocking I/O.
    """
    return _client.fetch_sync("day_gainers", count=count, start=start)