You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
217 lines
8.0 KiB
Python
217 lines
8.0 KiB
Python
"""
|
|
Direct Yahoo Finance predefined screener HTTP client.
|
|
|
|
Uses curl_cffi with browser fingerprint rotation (round-robin session pool)
|
|
to avoid 429 rate limiting — no yfinance_plus dependency.
|
|
|
|
Endpoint: https://query1.finance.yahoo.com/v1/finance/screener/predefined/saved
|
|
"""
|
|
import logging
|
|
import random
|
|
import threading
|
|
import time
|
|
from typing import Optional
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_BASE_URL = "https://query1.finance.yahoo.com/v1/finance/screener/predefined/saved"
|
|
|
|
_BROWSER_PROFILES = [
|
|
{
|
|
"impersonate": "chrome120",
|
|
"user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
"sec_ch_ua": '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
|
|
"sec_ch_ua_platform": '"macOS"',
|
|
},
|
|
{
|
|
"impersonate": "chrome110",
|
|
"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36",
|
|
"sec_ch_ua": '"Not A;Brand";v="8", "Chromium";v="110", "Google Chrome";v="110"',
|
|
"sec_ch_ua_platform": '"Windows"',
|
|
},
|
|
{
|
|
"impersonate": "edge99",
|
|
"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36 Edg/99.0.1150.36",
|
|
"sec_ch_ua": '"Not A;Brand";v="99", "Chromium";v="99", "Microsoft Edge";v="99"',
|
|
"sec_ch_ua_platform": '"Windows"',
|
|
},
|
|
{
|
|
"impersonate": "safari15_5",
|
|
"user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15",
|
|
"sec_ch_ua": None,
|
|
"sec_ch_ua_platform": None,
|
|
},
|
|
]
|
|
|
|
_ACCEPT_LANGUAGES = [
|
|
"en-US,en;q=0.9",
|
|
"en-US,en;q=0.9,ko;q=0.8",
|
|
"en-GB,en;q=0.9",
|
|
]
|
|
|
|
# Exchange code → friendly name (same mapping as screener_service)
|
|
_EXCHANGE_MAP = {
|
|
"NYQ": "NYSE", "NMS": "NASDAQ", "NGM": "NASDAQ",
|
|
"NCM": "NASDAQ", "ASE": "AMEX", "PCX": "NYSE_ARCA",
|
|
}
|
|
|
|
|
|
def _parse_quote(q: dict) -> dict:
|
|
"""Normalize a raw Yahoo Finance quote dict to our standard format."""
|
|
code = q.get("exchange", "")
|
|
volume = q.get("regularMarketVolume")
|
|
avg_vol = q.get("averageDailyVolume3Month")
|
|
return {
|
|
"symbol": q.get("symbol", ""),
|
|
"name": q.get("shortName") or q.get("longName"),
|
|
"exchange": _EXCHANGE_MAP.get(code, code),
|
|
"exchange_code": code,
|
|
"quote_type": q.get("quoteType"),
|
|
"market_cap": q.get("marketCap"),
|
|
"price": q.get("regularMarketPrice"),
|
|
"change_percent": q.get("regularMarketChangePercent"),
|
|
"volume": int(volume) if volume is not None else None,
|
|
"avg_volume_3m": int(avg_vol) if avg_vol is not None else None,
|
|
"shares_outstanding": q.get("sharesOutstanding"),
|
|
"pe_ratio": q.get("trailingPE"),
|
|
"forward_pe": q.get("forwardPE"),
|
|
"eps_ttm": q.get("epsTrailingTwelveMonths"),
|
|
"dividend_yield": q.get("trailingAnnualDividendYield"),
|
|
"fifty_two_week_high": q.get("fiftyTwoWeekHigh"),
|
|
"fifty_two_week_low": q.get("fiftyTwoWeekLow"),
|
|
"book_value": q.get("bookValue"),
|
|
"price_to_book": q.get("priceToBook"),
|
|
"analyst_rating": q.get("averageAnalystRating"),
|
|
}
|
|
|
|
|
|
class _YahooScreenerClient:
|
|
_POOL_SIZE = 4
|
|
_MAX_RETRIES = 4
|
|
|
|
def __init__(self):
|
|
self._lock = threading.Lock()
|
|
self._idx = 0
|
|
self._sessions: list = []
|
|
self._initialized = False
|
|
|
|
def _ensure_init(self):
|
|
if self._initialized:
|
|
return
|
|
with self._lock:
|
|
if self._initialized:
|
|
return
|
|
profiles = random.sample(_BROWSER_PROFILES, min(self._POOL_SIZE, len(_BROWSER_PROFILES)))
|
|
self._sessions = [self._make_session(p) for p in profiles]
|
|
self._initialized = True
|
|
|
|
def _make_session(self, profile: dict):
|
|
from curl_cffi import requests as cr
|
|
try:
|
|
session = cr.Session(impersonate=profile["impersonate"])
|
|
except Exception:
|
|
session = cr.Session(impersonate="chrome")
|
|
|
|
headers = {
|
|
"User-Agent": profile["user_agent"],
|
|
"Accept": "application/json, text/plain, */*",
|
|
"Accept-Language": random.choice(_ACCEPT_LANGUAGES),
|
|
"Accept-Encoding": "gzip, deflate, br",
|
|
"Cache-Control": "no-cache",
|
|
"Origin": "https://finance.yahoo.com",
|
|
"Referer": "https://finance.yahoo.com/screener/predefined/day_gainers/",
|
|
}
|
|
if profile.get("sec_ch_ua"):
|
|
headers["Sec-Ch-Ua"] = profile["sec_ch_ua"]
|
|
headers["Sec-Ch-Ua-Mobile"] = "?0"
|
|
headers["Sec-Ch-Ua-Platform"] = profile["sec_ch_ua_platform"]
|
|
headers["Sec-Fetch-Dest"] = "empty"
|
|
headers["Sec-Fetch-Mode"] = "cors"
|
|
headers["Sec-Fetch-Site"] = "same-site"
|
|
session.headers.update(headers)
|
|
return session
|
|
|
|
def _next_session(self):
|
|
with self._lock:
|
|
s = self._sessions[self._idx % len(self._sessions)]
|
|
self._idx += 1
|
|
return s
|
|
|
|
def _rotate(self):
|
|
"""Replace the next slot with a fresh session on a random profile."""
|
|
profile = random.choice(_BROWSER_PROFILES)
|
|
new_s = self._make_session(profile)
|
|
with self._lock:
|
|
self._sessions[self._idx % len(self._sessions)] = new_s
|
|
|
|
def _rotate_used(self):
|
|
"""Replace the slot that was just used (idx already incremented after _next_session)."""
|
|
profile = random.choice(_BROWSER_PROFILES)
|
|
new_s = self._make_session(profile)
|
|
with self._lock:
|
|
slot = (self._idx - 1) % len(self._sessions)
|
|
self._sessions[slot] = new_s
|
|
|
|
def fetch_sync(
|
|
self,
|
|
preset: str = "day_gainers",
|
|
count: int = 200,
|
|
start: int = 0,
|
|
) -> tuple[list, int, Optional[str]]:
|
|
"""
|
|
Synchronous fetch — run via asyncio.run_in_executor.
|
|
|
|
Returns (quotes: list[dict], total: int, error: str|None).
|
|
"""
|
|
self._ensure_init()
|
|
params = {
|
|
"formatted": "false",
|
|
"lang": "en-US",
|
|
"region": "US",
|
|
"scrIds": preset,
|
|
"count": count,
|
|
"start": start,
|
|
}
|
|
last_exc: Exception = RuntimeError("no attempts made")
|
|
for attempt in range(self._MAX_RETRIES):
|
|
if attempt > 0:
|
|
delay = 2 ** attempt + random.uniform(0, 1)
|
|
logger.info("[YahooClient] backoff %.1fs (attempt %d)", delay, attempt)
|
|
time.sleep(delay)
|
|
self._rotate()
|
|
|
|
session = self._next_session()
|
|
try:
|
|
resp = session.get(_BASE_URL, params=params, timeout=15)
|
|
if resp.status_code in (429, 401):
|
|
logger.warning("[YahooClient] HTTP %d — rotating session", resp.status_code)
|
|
self._rotate()
|
|
last_exc = RuntimeError(f"HTTP {resp.status_code}")
|
|
continue
|
|
resp.raise_for_status()
|
|
result = (resp.json().get("finance", {}).get("result") or [{}])[0]
|
|
quotes = result.get("quotes", [])
|
|
total = result.get("total", len(quotes))
|
|
# Always replace the just-used session so each 5-min fetch starts fresh
|
|
self._rotate_used()
|
|
return quotes, total, None
|
|
except Exception as e:
|
|
last_exc = e
|
|
if any(x in str(e) for x in ("429", "401", "rate")):
|
|
self._rotate()
|
|
logger.warning("[YahooClient] attempt %d: %s", attempt, e)
|
|
|
|
return [], 0, str(last_exc)
|
|
|
|
|
|
# Module-level singleton — lazy-initialized on first call
|
|
_client = _YahooScreenerClient()
|
|
|
|
|
|
def fetch_day_gainers_sync(count: int = 200, start: int = 0) -> tuple[list, int, Optional[str]]:
|
|
"""
|
|
Fetch day_gainers from Yahoo Finance. Returns (raw_quotes, total, error).
|
|
Call via asyncio.run_in_executor — blocking I/O.
|
|
"""
|
|
return _client.fetch_sync("day_gainers", count=count, start=start)
|