refactor: day_gainers → Yahoo Finance API 직접 호출 (curl_cffi 우회)
- yfinance_plus/preset 제거, query1.finance.yahoo.com 직접 호출 - curl_cffi 세션 풀(4개) + 브라우저 지문 로테이션으로 429 우회 - count=200 반환 확인 (yfinance preset 25개 한계 해소) - /stocks/gainers: page/page_size → count 파라미터로 변경 - collector: fetch_day_gainers_sync(200) 사용 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>main
I Luk Kim
3 months ago
parent
1d087b54a4
commit
cc024ecbe1
@ -0,0 +1,206 @@
|
||||
"""
|
||||
Direct Yahoo Finance predefined screener HTTP client.
|
||||
|
||||
Uses curl_cffi with browser fingerprint rotation (round-robin session pool)
|
||||
to avoid 429 rate limiting — no yfinance_plus dependency.
|
||||
|
||||
Endpoint: https://query1.finance.yahoo.com/v1/finance/screener/predefined/saved
|
||||
"""
|
||||
import logging
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_BASE_URL = "https://query1.finance.yahoo.com/v1/finance/screener/predefined/saved"
|
||||
|
||||
_BROWSER_PROFILES = [
|
||||
{
|
||||
"impersonate": "chrome120",
|
||||
"user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"sec_ch_ua": '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
|
||||
"sec_ch_ua_platform": '"macOS"',
|
||||
},
|
||||
{
|
||||
"impersonate": "chrome110",
|
||||
"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36",
|
||||
"sec_ch_ua": '"Not A;Brand";v="8", "Chromium";v="110", "Google Chrome";v="110"',
|
||||
"sec_ch_ua_platform": '"Windows"',
|
||||
},
|
||||
{
|
||||
"impersonate": "edge99",
|
||||
"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36 Edg/99.0.1150.36",
|
||||
"sec_ch_ua": '"Not A;Brand";v="99", "Chromium";v="99", "Microsoft Edge";v="99"',
|
||||
"sec_ch_ua_platform": '"Windows"',
|
||||
},
|
||||
{
|
||||
"impersonate": "safari15_5",
|
||||
"user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15",
|
||||
"sec_ch_ua": None,
|
||||
"sec_ch_ua_platform": None,
|
||||
},
|
||||
]
|
||||
|
||||
_ACCEPT_LANGUAGES = [
|
||||
"en-US,en;q=0.9",
|
||||
"en-US,en;q=0.9,ko;q=0.8",
|
||||
"en-GB,en;q=0.9",
|
||||
]
|
||||
|
||||
# Exchange code → friendly name (same mapping as screener_service)
|
||||
_EXCHANGE_MAP = {
|
||||
"NYQ": "NYSE", "NMS": "NASDAQ", "NGM": "NASDAQ",
|
||||
"NCM": "NASDAQ", "ASE": "AMEX", "PCX": "NYSE_ARCA",
|
||||
}
|
||||
|
||||
|
||||
def _parse_quote(q: dict) -> dict:
|
||||
"""Normalize a raw Yahoo Finance quote dict to our standard format."""
|
||||
code = q.get("exchange", "")
|
||||
volume = q.get("regularMarketVolume")
|
||||
avg_vol = q.get("averageDailyVolume3Month")
|
||||
return {
|
||||
"symbol": q.get("symbol", ""),
|
||||
"name": q.get("shortName") or q.get("longName"),
|
||||
"exchange": _EXCHANGE_MAP.get(code, code),
|
||||
"exchange_code": code,
|
||||
"quote_type": q.get("quoteType"),
|
||||
"market_cap": q.get("marketCap"),
|
||||
"price": q.get("regularMarketPrice"),
|
||||
"change_percent": q.get("regularMarketChangePercent"),
|
||||
"volume": int(volume) if volume is not None else None,
|
||||
"avg_volume_3m": int(avg_vol) if avg_vol is not None else None,
|
||||
"shares_outstanding": q.get("sharesOutstanding"),
|
||||
"pe_ratio": q.get("trailingPE"),
|
||||
"forward_pe": q.get("forwardPE"),
|
||||
"eps_ttm": q.get("epsTrailingTwelveMonths"),
|
||||
"dividend_yield": q.get("trailingAnnualDividendYield"),
|
||||
"fifty_two_week_high": q.get("fiftyTwoWeekHigh"),
|
||||
"fifty_two_week_low": q.get("fiftyTwoWeekLow"),
|
||||
"book_value": q.get("bookValue"),
|
||||
"price_to_book": q.get("priceToBook"),
|
||||
"analyst_rating": q.get("averageAnalystRating"),
|
||||
}
|
||||
|
||||
|
||||
class _YahooScreenerClient:
|
||||
_POOL_SIZE = 4
|
||||
_MAX_RETRIES = 4
|
||||
|
||||
def __init__(self):
|
||||
self._lock = threading.Lock()
|
||||
self._idx = 0
|
||||
self._sessions: list = []
|
||||
self._initialized = False
|
||||
|
||||
def _ensure_init(self):
|
||||
if self._initialized:
|
||||
return
|
||||
with self._lock:
|
||||
if self._initialized:
|
||||
return
|
||||
profiles = random.sample(_BROWSER_PROFILES, min(self._POOL_SIZE, len(_BROWSER_PROFILES)))
|
||||
self._sessions = [self._make_session(p) for p in profiles]
|
||||
self._initialized = True
|
||||
|
||||
def _make_session(self, profile: dict):
|
||||
from curl_cffi import requests as cr
|
||||
try:
|
||||
session = cr.Session(impersonate=profile["impersonate"])
|
||||
except Exception:
|
||||
session = cr.Session(impersonate="chrome")
|
||||
|
||||
headers = {
|
||||
"User-Agent": profile["user_agent"],
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"Accept-Language": random.choice(_ACCEPT_LANGUAGES),
|
||||
"Accept-Encoding": "gzip, deflate, br",
|
||||
"Cache-Control": "no-cache",
|
||||
"Origin": "https://finance.yahoo.com",
|
||||
"Referer": "https://finance.yahoo.com/screener/predefined/day_gainers/",
|
||||
}
|
||||
if profile.get("sec_ch_ua"):
|
||||
headers["Sec-Ch-Ua"] = profile["sec_ch_ua"]
|
||||
headers["Sec-Ch-Ua-Mobile"] = "?0"
|
||||
headers["Sec-Ch-Ua-Platform"] = profile["sec_ch_ua_platform"]
|
||||
headers["Sec-Fetch-Dest"] = "empty"
|
||||
headers["Sec-Fetch-Mode"] = "cors"
|
||||
headers["Sec-Fetch-Site"] = "same-site"
|
||||
session.headers.update(headers)
|
||||
return session
|
||||
|
||||
def _next_session(self):
|
||||
with self._lock:
|
||||
s = self._sessions[self._idx % len(self._sessions)]
|
||||
self._idx += 1
|
||||
return s
|
||||
|
||||
def _rotate(self):
|
||||
"""Replace the next slot with a fresh session on a random profile."""
|
||||
profile = random.choice(_BROWSER_PROFILES)
|
||||
new_s = self._make_session(profile)
|
||||
with self._lock:
|
||||
self._sessions[self._idx % len(self._sessions)] = new_s
|
||||
|
||||
def fetch_sync(
|
||||
self,
|
||||
preset: str = "day_gainers",
|
||||
count: int = 200,
|
||||
start: int = 0,
|
||||
) -> tuple[list, int, Optional[str]]:
|
||||
"""
|
||||
Synchronous fetch — run via asyncio.run_in_executor.
|
||||
|
||||
Returns (quotes: list[dict], total: int, error: str|None).
|
||||
"""
|
||||
self._ensure_init()
|
||||
params = {
|
||||
"formatted": "false",
|
||||
"lang": "en-US",
|
||||
"region": "US",
|
||||
"scrIds": preset,
|
||||
"count": count,
|
||||
"start": start,
|
||||
}
|
||||
last_exc: Exception = RuntimeError("no attempts made")
|
||||
for attempt in range(self._MAX_RETRIES):
|
||||
if attempt > 0:
|
||||
delay = 2 ** attempt + random.uniform(0, 1)
|
||||
logger.info("[YahooClient] backoff %.1fs (attempt %d)", delay, attempt)
|
||||
time.sleep(delay)
|
||||
self._rotate()
|
||||
|
||||
session = self._next_session()
|
||||
try:
|
||||
resp = session.get(_BASE_URL, params=params, timeout=15)
|
||||
if resp.status_code in (429, 401):
|
||||
logger.warning("[YahooClient] HTTP %d — rotating session", resp.status_code)
|
||||
self._rotate()
|
||||
last_exc = RuntimeError(f"HTTP {resp.status_code}")
|
||||
continue
|
||||
resp.raise_for_status()
|
||||
result = (resp.json().get("finance", {}).get("result") or [{}])[0]
|
||||
quotes = result.get("quotes", [])
|
||||
total = result.get("total", len(quotes))
|
||||
return quotes, total, None
|
||||
except Exception as e:
|
||||
last_exc = e
|
||||
if any(x in str(e) for x in ("429", "401", "rate")):
|
||||
self._rotate()
|
||||
logger.warning("[YahooClient] attempt %d: %s", attempt, e)
|
||||
|
||||
return [], 0, str(last_exc)
|
||||
|
||||
|
||||
# Module-level singleton — lazy-initialized on first call
|
||||
_client = _YahooScreenerClient()
|
||||
|
||||
|
||||
def fetch_day_gainers_sync(count: int = 200, start: int = 0) -> tuple[list, int, Optional[str]]:
|
||||
"""
|
||||
Fetch day_gainers from Yahoo Finance. Returns (raw_quotes, total, error).
|
||||
Call via asyncio.run_in_executor — blocking I/O.
|
||||
"""
|
||||
return _client.fetch_sync("day_gainers", count=count, start=start)
|
||||
Loading…
Reference in New Issue