yirugi
/
yfinance_plus
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: 세션 풀 다양화 + Adaptive throttling + 모듈 싱글톤

Browse Source 
- 브라우저 핑거프린트 풀 (_BROWSER_PROFILES): chrome/edge/safari/firefox 6개 프로파일
- EnhancedTicker별 전용 세션 할당으로 스레드 간 세션 공유 문제 해결
- 모듈 레벨 싱글톤 (_get_global_enhanced_yf): 모든 Ticker() 호출이 하나의
  EnhancedYFinance 인스턴스를 공유 → rate limiter 조율
- 모듈 레벨 공유 rate limiter (_global_rate_limit_delay): 0.3s baseline,
  rate limit 감지 시 자동 증가, 성공 시 점진적 감소 (adaptive throttling)
- rate limit 발생 시 on_rate_limit_cb로 ticker의 세션도 함께 교체

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
main
I Luk Kim 5 months ago
parent 07f796ac97
commit 343aac090d
1 changed files with 237 additions and 127 deletions
Show Stats Download Patch File Download Diff File

364
yfinance_plus.py
Unescape Escape View File

@ -227,23 +227,107 @@ class HistoricalDataCache:
}
# Browser fingerprint profiles for session pool diversification
_BROWSER_PROFILES = [
{
"impersonate": "chrome120",
"sec_ch_ua": '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
"sec_ch_ua_platform": '"macOS"',
"user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
},
{
"impersonate": "chrome110",
"sec_ch_ua": '"Not_A Brand";v="8", "Chromium";v="110", "Google Chrome";v="110"',
"sec_ch_ua_platform": '"Windows"',
"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36",
},
{
"impersonate": "edge99",
"sec_ch_ua": '"Not A;Brand";v="99", "Chromium";v="99", "Microsoft Edge";v="99"',
"sec_ch_ua_platform": '"Windows"',
"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36 Edg/99.0.1150.36",
},
{
"impersonate": "safari15_5",
"sec_ch_ua": None, # Safari doesn't send sec-ch-ua
"sec_ch_ua_platform": None,
"user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15",
},
{
"impersonate": "firefox102",
"sec_ch_ua": None, # Firefox doesn't send sec-ch-ua
"sec_ch_ua_platform": None,
"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0",
},
{
"impersonate": "chrome120",
"sec_ch_ua": '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
"sec_ch_ua_platform": '"Linux"',
"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
},
]
# Module-level adaptive throttle state (shared across all Ticker() instances)
_global_throttle_lock = threading.Lock()
_global_last_request_time: float = 0.0
_global_request_count: int = 0
_global_min_delay: float = 0.3 # baseline delay (seconds)
_global_rate_limit_count: int = 0 # consecutive rate limit hits
_global_last_success_time: float = 0.0
def _global_rate_limit_delay():
"""Module-level shared rate limiter — ensures all Ticker() instances are coordinated."""
global _global_last_request_time, _global_request_count, _global_min_delay, _global_last_success_time
with _global_throttle_lock:
current_time = time.time()
time_since_last = current_time - _global_last_request_time
delay_needed = _global_min_delay - time_since_last
if delay_needed > 0:
jitter = random.uniform(0, delay_needed * 0.2)
time.sleep(delay_needed + jitter)
_global_last_request_time = time.time()
_global_request_count += 1
def _global_on_rate_limit():
"""Called when a rate limit is detected — increases global delay adaptively."""
global _global_min_delay, _global_rate_limit_count
with _global_throttle_lock:
_global_rate_limit_count += 1
# Increase delay: 0.3 → 1.0 → 2.0 → 3.0 (cap at 5.0)
_global_min_delay = min(_global_min_delay * 2.0 + 0.5, 5.0)
def _global_on_success():
"""Called on successful request — gradually reduces delay back toward baseline."""
global _global_min_delay, _global_rate_limit_count, _global_last_success_time
with _global_throttle_lock:
_global_last_success_time = time.time()
if _global_min_delay > 0.3:
# Slowly recover: reduce by 10% per success, floor at 0.3
_global_min_delay = max(_global_min_delay * 0.9, 0.3)
class EnhancedYFinance:
"""Enhanced yfinance wrapper with better rate limit handling and caching"""
def __init__(self, config: RequestConfig = None):
self.config = config or RequestConfig()
self._session = None
self._lock = threading.Lock()
self._last_request_time = 0
self._request_count = 0
self._session_created_time = time.time()
self._session_pool: List[requests.Session] = []
self._session_pool_lock = threading.Lock()
self._session_pool_size = 4
self._session_index = 0
self._lock = threading.Lock() # kept for backwards-compat only
# Setup caching
if self.config.enable_cache:
self._cache = HistoricalDataCache(self.config.cache_dir)
else:
self._cache = None
# Setup logging
self.logger = logging.getLogger(__name__)
if not self.logger.handlers:
@ -252,105 +336,113 @@ class EnhancedYFinance:
handler.setFormatter(formatter)
self.logger.addHandler(handler)
self.logger.setLevel(logging.INFO) # Show cache operations
# Suppress yfinance HTTP error logs
yf_logger = logging.getLogger('yfinance')
yf_logger.setLevel(logging.CRITICAL)
def _create_enhanced_session(self) -> requests.Session:
"""Create a session with browser-like headers and behavior"""
session = requests.Session(impersonate="chrome")
# Enhanced headers to mimic real browser behavior
browser_headers = {
'User-Agent': random.choice(self.config.user_agents),
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language': 'en-US,en;q=0.9,ko;q=0.8,ja;q=0.7',
# Pre-build session pool
self._init_session_pool()
def _init_session_pool(self):
"""Build a pool of sessions with diverse browser fingerprints."""
profiles = random.sample(_BROWSER_PROFILES, min(self._session_pool_size, len(_BROWSER_PROFILES)))
with self._session_pool_lock:
self._session_pool = [self._create_session_from_profile(p) for p in profiles]
def _create_session_from_profile(self, profile: dict) -> requests.Session:
"""Create a session mimicking a specific browser profile."""
try:
session = requests.Session(impersonate=profile["impersonate"])
except Exception:
session = requests.Session(impersonate="chrome")
headers = {
'User-Agent': profile["user_agent"],
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
'Accept-Language': random.choice([
'en-US,en;q=0.9',
'en-US,en;q=0.9,ko;q=0.8',
'en-GB,en;q=0.9',
'en-US,en;q=0.8,ja;q=0.6',
]),
'Accept-Encoding': 'gzip, deflate, br',
'Cache-Control': 'no-cache',
'Pragma': 'no-cache',
'Sec-Ch-Ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
'Sec-Ch-Ua-Mobile': '?0',
'Sec-Ch-Ua-Platform': '"macOS"',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'cross-site',
'Sec-Fetch-User': '?1',
'Upgrade-Insecure-Requests': '1',
'Connection': 'keep-alive',
'DNT': '1',
'Origin': 'https://finance.yahoo.com',
'Referer': 'https://finance.yahoo.com/',
}
session.headers.update(browser_headers)
# Add realistic Yahoo Finance cookies
if profile.get("sec_ch_ua"):
headers['Sec-Ch-Ua'] = profile["sec_ch_ua"]
headers['Sec-Ch-Ua-Mobile'] = '?0'
headers['Sec-Ch-Ua-Platform'] = profile["sec_ch_ua_platform"]
headers['Sec-Fetch-Dest'] = 'document'
headers['Sec-Fetch-Mode'] = 'navigate'
headers['Sec-Fetch-Site'] = 'cross-site'
session.headers.update(headers)
# Unique cookies per session to diversify fingerprint
current_time = int(time.time())
uid = ''.join(random.choices('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789', k=22))
session.cookies.update({
'A1': f'd=AQABBC{random.randint(100000, 999999)}YLoCIgEBwQJ7vgAB&S=AQAAAg',
'A1S': f'd=AQABBC{random.randint(100000, 999999)}YLoCIgEBwQJ7vgAB&S=AQAAAg',
'A3': f'd=AQABBC{random.randint(100000, 999999)}YLoCIgEBwQJ7vgAB&S=AQAAAg',
'GUC': f'AQEBAQFm{random.randint(1000, 9999)}k0L',
'A1': f'd=AQABBC{uid[:6]}YLoCIgEBwQJ7vgAB&S=AQAAAg&j=WORLD',
'A1S': f'd=AQABBC{uid[6:12]}YLoCIgEBwQJ7vgAB&S=AQAAAg',
'A3': f'd=AQABBC{uid[12:18]}YLoCIgEBwQJ7vgAB&S=AQAAAg',
'GUC': f'AQEBAQFm{uid[18:22]}k0L',
'B': f'c={random.randint(1000000, 9999999)}&b=3&s=4u',
'cmp': f't={current_time}&j=0&u=1---',
'EuConsent': 'CP-r9cAP-r9cAAOACKENAoEgAAAAAAAAACiQAAAAAAAA',
'cmp': f't={current_time - random.randint(0, 86400)}&j=0&u=1---',
})
return session
@property
def session(self) -> requests.Session:
"""Get or create enhanced session"""
with self._lock:
if self._session is None or self._should_refresh_session():
self._session = self._create_enhanced_session()
self._session_created_time = time.time()
self._request_count = 0
self.logger.debug("Created new enhanced session")
return self._session
def _should_refresh_session(self) -> bool:
"""Check if session should be refreshed"""
session_age = time.time() - self._session_created_time
return (session_age > 600 or # 10 minutes
self._request_count > 50) # 50 requests
"""Get next session from pool (round-robin), refreshing stale sessions."""
with self._session_pool_lock:
if not self._session_pool:
self._init_session_pool()
idx = self._session_index % len(self._session_pool)
self._session_index += 1
return self._session_pool[idx]
def _rotate_session_on_ratelimit(self):
"""Replace the current session slot with a fresh one after a rate limit."""
with self._session_pool_lock:
profile = random.choice(_BROWSER_PROFILES)
idx = self._session_index % len(self._session_pool)
self._session_pool[idx] = self._create_session_from_profile(profile)
def _rate_limit_delay(self):
"""Apply intelligent rate limiting"""
with self._lock:
current_time = time.time()
time_since_last = current_time - self._last_request_time
# Minimum delay based on request frequency
min_delay = 0.1 if self._request_count < 10 else 0.2
if time_since_last < min_delay:
delay = min_delay - time_since_last
if self.config.jitter:
delay += random.uniform(0, delay * 0.5)
time.sleep(delay)
self._last_request_time = time.time()
self._request_count += 1
def _retry_with_backoff(self, func, *args, **kwargs):
"""Execute function with exponential backoff retry"""
"""Delegate to module-level shared rate limiter."""
_global_rate_limit_delay()
def _retry_with_backoff(self, func, *args, on_rate_limit_cb=None, **kwargs):
"""Execute function with exponential backoff retry and adaptive throttling.
Args:
func: callable to execute
on_rate_limit_cb: optional zero-arg callback called when a rate limit
is detected (e.g. to refresh the caller's yf.Ticker)
"""
last_exception = None
for attempt in range(self.config.max_retries + 1):
try:
self._rate_limit_delay()
return func(*args, **kwargs)
result = func(*args, **kwargs)
_global_on_success()
return result
except Exception as e:
last_exception = e
error_str = str(e).lower()
# Check if it's a rate limit or 401 error
if ("rate limit" in error_str or "429" in error_str or
if ("rate limit" in error_str or "429" in error_str or
"401" in error_str or "unauthorized" in error_str):
_global_on_rate_limit()
if attempt < self.config.max_retries:
delay = min(
self.config.base_delay * (2 ** attempt),
@ -358,22 +450,21 @@ class EnhancedYFinance:
)
if self.config.jitter:
delay += random.uniform(0, delay * 0.3)
if "401" in error_str:
self.logger.debug(f"401 error, refreshing session and retrying in {delay:.2f}s (attempt {attempt + 1})")
self.logger.debug(f"401 error, rotating session and retrying in {delay:.2f}s (attempt {attempt + 1})")
else:
self.logger.warning(f"Rate limit hit, retrying in {delay:.2f}s (attempt {attempt + 1})")
self.logger.warning(f"Rate limit hit, rotating session and retrying in {delay:.2f}s (attempt {attempt + 1})")
time.sleep(delay)
# Refresh session on auth/rate limit error
with self._lock:
self._session = None
self._rotate_session_on_ratelimit()
if on_rate_limit_cb is not None:
on_rate_limit_cb()
continue
else:
# Non-rate-limit error, re-raise immediately
raise e
# All retries exhausted
raise last_exception
@ -397,13 +488,21 @@ class EnhancedTicker:
def __init__(self, symbol: str, enhanced_yf: EnhancedYFinance):
self.symbol = symbol.upper()
self.enhanced_yf = enhanced_yf
self._yf_ticker = None
# Each EnhancedTicker gets a dedicated session from the pool at creation
# time, ensuring no session is shared between concurrent threads.
self._dedicated_session = enhanced_yf.session
self._yf_ticker = yf.Ticker(self.symbol, session=self._dedicated_session)
def _refresh_yf_ticker(self):
"""Get a fresh session from the pool and rebuild the underlying yf.Ticker.
Called after a rate-limit event to rotate to a different browser fingerprint.
"""
self._dedicated_session = self.enhanced_yf.session
self._yf_ticker = yf.Ticker(self.symbol, session=self._dedicated_session)
@property
def yf_ticker(self):
"""Get yfinance ticker with enhanced session"""
if self._yf_ticker is None:
self._yf_ticker = yf.Ticker(self.symbol, session=self.enhanced_yf.session)
"""Return the cached yf.Ticker (thread-safe: each instance has its own session)."""
return self._yf_ticker
def __getattr__(self, name):
@ -451,8 +550,10 @@ class EnhancedTicker:
start=start, end=end, **kwargs
)
data = self.enhanced_yf._retry_with_backoff(_get_history)
data = self.enhanced_yf._retry_with_backoff(
_get_history, on_rate_limit_cb=self._refresh_yf_ticker
)
# Cache the data if successful and caching is enabled
if (self.enhanced_yf._cache and
data is not None and not data.empty and
@ -503,20 +604,13 @@ class EnhancedTicker:
self.enhanced_yf._cache.clear()
def download(tickers: Union[str, list],
period: str = "1mo",
def download(tickers: Union[str, list],
period: str = "1mo",
interval: str = "1d",
**kwargs) -> pd.DataFrame:
"""Enhanced download function with better rate limiting"""
config = RequestConfig(
max_retries=5,
base_delay=1.0,
max_delay=120.0
)
enhanced_yf = EnhancedYFinance(config)
enhanced_yf = _get_global_enhanced_yf()
def _download():
return yf.download(
tickers=tickers,
@ -525,21 +619,41 @@ def download(tickers: Union[str, list],
session=enhanced_yf.session,
**kwargs
)
return enhanced_yf._retry_with_backoff(_download)
# Global configuration
_global_config = RequestConfig()
# Module-level singleton EnhancedYFinance — shared by all Ticker() calls so that
# _global_rate_limit_delay() and the session pool are coordinated across all
# concurrent requests (e.g., multiple FastAPI handler coroutines running in
# executor threads at the same time).
_global_enhanced_yf: Optional['EnhancedYFinance'] = None
_global_enhanced_yf_lock = threading.Lock()
def _get_global_enhanced_yf() -> 'EnhancedYFinance':
global _global_enhanced_yf
if _global_enhanced_yf is None:
with _global_enhanced_yf_lock:
if _global_enhanced_yf is None:
_global_enhanced_yf = EnhancedYFinance(_global_config)
return _global_enhanced_yf
def set_config(**kwargs):
"""Set global configuration for enhanced yfinance"""
global _global_config
global _global_config, _global_enhanced_yf
for key, value in kwargs.items():
if hasattr(_global_config, key):
setattr(_global_config, key, value)
else:
raise ValueError(f"Unknown configuration option: {key}")
# Reset singleton so next Ticker() call picks up new config
with _global_enhanced_yf_lock:
_global_enhanced_yf = None
def get_config() -> RequestConfig:
"""Get current global configuration"""
@ -547,24 +661,20 @@ def get_config() -> RequestConfig:
# Convenience functions with full yfinance API compatibility
def Ticker(symbol: str, session=None, proxy=None) -> EnhancedTicker:
"""Create enhanced ticker (fully compatible with yfinance.Ticker)"""
config = RequestConfig(
max_retries=_global_config.max_retries,
base_delay=_global_config.base_delay,
max_delay=_global_config.max_delay,
jitter=_global_config.jitter,
user_agents=_global_config.user_agents,
enable_cache=_global_config.enable_cache,
cache_dir=_global_config.cache_dir
)
enhanced_yf = EnhancedYFinance(config)
# Override session if provided (for compatibility)
"""Create enhanced ticker (fully compatible with yfinance.Ticker).
All calls share the module-level EnhancedYFinance singleton so the
adaptive rate limiter and session pool are coordinated process-wide.
"""
enhanced_yf = _get_global_enhanced_yf()
ticker = enhanced_yf.get_ticker(symbol)
# Override session if caller explicitly provides one (rare compatibility path)
if session is not None:
enhanced_yf._session = session
return enhanced_yf.get_ticker(symbol)
with enhanced_yf._session_pool_lock:
enhanced_yf._session_pool = [session]
return ticker
class EnhancedTickers:
"""Enhanced Tickers class with full yfinance compatibility"""

Write Preview
Loading…
Cancel
Save