diff --git a/yfinance_plus.py b/yfinance_plus.py index f9d9590..613c20c 100644 --- a/yfinance_plus.py +++ b/yfinance_plus.py @@ -227,23 +227,107 @@ class HistoricalDataCache: } +# Browser fingerprint profiles for session pool diversification +_BROWSER_PROFILES = [ + { + "impersonate": "chrome120", + "sec_ch_ua": '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"', + "sec_ch_ua_platform": '"macOS"', + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + }, + { + "impersonate": "chrome110", + "sec_ch_ua": '"Not_A Brand";v="8", "Chromium";v="110", "Google Chrome";v="110"', + "sec_ch_ua_platform": '"Windows"', + "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36", + }, + { + "impersonate": "edge99", + "sec_ch_ua": '"Not A;Brand";v="99", "Chromium";v="99", "Microsoft Edge";v="99"', + "sec_ch_ua_platform": '"Windows"', + "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36 Edg/99.0.1150.36", + }, + { + "impersonate": "safari15_5", + "sec_ch_ua": None, # Safari doesn't send sec-ch-ua + "sec_ch_ua_platform": None, + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15", + }, + { + "impersonate": "firefox102", + "sec_ch_ua": None, # Firefox doesn't send sec-ch-ua + "sec_ch_ua_platform": None, + "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0", + }, + { + "impersonate": "chrome120", + "sec_ch_ua": '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"', + "sec_ch_ua_platform": '"Linux"', + "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + }, +] + +# Module-level adaptive throttle state (shared across all Ticker() instances) +_global_throttle_lock = threading.Lock() +_global_last_request_time: float = 0.0 +_global_request_count: int = 0 +_global_min_delay: float = 0.3 # baseline delay (seconds) +_global_rate_limit_count: int = 0 # consecutive rate limit hits +_global_last_success_time: float = 0.0 + + +def _global_rate_limit_delay(): + """Module-level shared rate limiter — ensures all Ticker() instances are coordinated.""" + global _global_last_request_time, _global_request_count, _global_min_delay, _global_last_success_time + with _global_throttle_lock: + current_time = time.time() + time_since_last = current_time - _global_last_request_time + + delay_needed = _global_min_delay - time_since_last + if delay_needed > 0: + jitter = random.uniform(0, delay_needed * 0.2) + time.sleep(delay_needed + jitter) + + _global_last_request_time = time.time() + _global_request_count += 1 + + +def _global_on_rate_limit(): + """Called when a rate limit is detected — increases global delay adaptively.""" + global _global_min_delay, _global_rate_limit_count + with _global_throttle_lock: + _global_rate_limit_count += 1 + # Increase delay: 0.3 → 1.0 → 2.0 → 3.0 (cap at 5.0) + _global_min_delay = min(_global_min_delay * 2.0 + 0.5, 5.0) + + +def _global_on_success(): + """Called on successful request — gradually reduces delay back toward baseline.""" + global _global_min_delay, _global_rate_limit_count, _global_last_success_time + with _global_throttle_lock: + _global_last_success_time = time.time() + if _global_min_delay > 0.3: + # Slowly recover: reduce by 10% per success, floor at 0.3 + _global_min_delay = max(_global_min_delay * 0.9, 0.3) + + class EnhancedYFinance: """Enhanced yfinance wrapper with better rate limit handling and caching""" - + def __init__(self, config: RequestConfig = None): self.config = config or RequestConfig() - self._session = None - self._lock = threading.Lock() - self._last_request_time = 0 - self._request_count = 0 - self._session_created_time = time.time() - + self._session_pool: List[requests.Session] = [] + self._session_pool_lock = threading.Lock() + self._session_pool_size = 4 + self._session_index = 0 + self._lock = threading.Lock() # kept for backwards-compat only + # Setup caching if self.config.enable_cache: self._cache = HistoricalDataCache(self.config.cache_dir) else: self._cache = None - + # Setup logging self.logger = logging.getLogger(__name__) if not self.logger.handlers: @@ -252,105 +336,113 @@ class EnhancedYFinance: handler.setFormatter(formatter) self.logger.addHandler(handler) self.logger.setLevel(logging.INFO) # Show cache operations - + # Suppress yfinance HTTP error logs yf_logger = logging.getLogger('yfinance') yf_logger.setLevel(logging.CRITICAL) - - def _create_enhanced_session(self) -> requests.Session: - """Create a session with browser-like headers and behavior""" - session = requests.Session(impersonate="chrome") - - # Enhanced headers to mimic real browser behavior - browser_headers = { - 'User-Agent': random.choice(self.config.user_agents), - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', - 'Accept-Language': 'en-US,en;q=0.9,ko;q=0.8,ja;q=0.7', + + # Pre-build session pool + self._init_session_pool() + + def _init_session_pool(self): + """Build a pool of sessions with diverse browser fingerprints.""" + profiles = random.sample(_BROWSER_PROFILES, min(self._session_pool_size, len(_BROWSER_PROFILES))) + with self._session_pool_lock: + self._session_pool = [self._create_session_from_profile(p) for p in profiles] + + def _create_session_from_profile(self, profile: dict) -> requests.Session: + """Create a session mimicking a specific browser profile.""" + try: + session = requests.Session(impersonate=profile["impersonate"]) + except Exception: + session = requests.Session(impersonate="chrome") + + headers = { + 'User-Agent': profile["user_agent"], + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', + 'Accept-Language': random.choice([ + 'en-US,en;q=0.9', + 'en-US,en;q=0.9,ko;q=0.8', + 'en-GB,en;q=0.9', + 'en-US,en;q=0.8,ja;q=0.6', + ]), 'Accept-Encoding': 'gzip, deflate, br', 'Cache-Control': 'no-cache', - 'Pragma': 'no-cache', - 'Sec-Ch-Ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"', - 'Sec-Ch-Ua-Mobile': '?0', - 'Sec-Ch-Ua-Platform': '"macOS"', - 'Sec-Fetch-Dest': 'document', - 'Sec-Fetch-Mode': 'navigate', - 'Sec-Fetch-Site': 'cross-site', - 'Sec-Fetch-User': '?1', - 'Upgrade-Insecure-Requests': '1', 'Connection': 'keep-alive', - 'DNT': '1', 'Origin': 'https://finance.yahoo.com', 'Referer': 'https://finance.yahoo.com/', } - - session.headers.update(browser_headers) - - # Add realistic Yahoo Finance cookies + if profile.get("sec_ch_ua"): + headers['Sec-Ch-Ua'] = profile["sec_ch_ua"] + headers['Sec-Ch-Ua-Mobile'] = '?0' + headers['Sec-Ch-Ua-Platform'] = profile["sec_ch_ua_platform"] + headers['Sec-Fetch-Dest'] = 'document' + headers['Sec-Fetch-Mode'] = 'navigate' + headers['Sec-Fetch-Site'] = 'cross-site' + + session.headers.update(headers) + + # Unique cookies per session to diversify fingerprint current_time = int(time.time()) + uid = ''.join(random.choices('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789', k=22)) session.cookies.update({ - 'A1': f'd=AQABBC{random.randint(100000, 999999)}YLoCIgEBwQJ7vgAB&S=AQAAAg', - 'A1S': f'd=AQABBC{random.randint(100000, 999999)}YLoCIgEBwQJ7vgAB&S=AQAAAg', - 'A3': f'd=AQABBC{random.randint(100000, 999999)}YLoCIgEBwQJ7vgAB&S=AQAAAg', - 'GUC': f'AQEBAQFm{random.randint(1000, 9999)}k0L', + 'A1': f'd=AQABBC{uid[:6]}YLoCIgEBwQJ7vgAB&S=AQAAAg&j=WORLD', + 'A1S': f'd=AQABBC{uid[6:12]}YLoCIgEBwQJ7vgAB&S=AQAAAg', + 'A3': f'd=AQABBC{uid[12:18]}YLoCIgEBwQJ7vgAB&S=AQAAAg', + 'GUC': f'AQEBAQFm{uid[18:22]}k0L', 'B': f'c={random.randint(1000000, 9999999)}&b=3&s=4u', - 'cmp': f't={current_time}&j=0&u=1---', - 'EuConsent': 'CP-r9cAP-r9cAAOACKENAoEgAAAAAAAAACiQAAAAAAAA', + 'cmp': f't={current_time - random.randint(0, 86400)}&j=0&u=1---', }) - return session - + @property def session(self) -> requests.Session: - """Get or create enhanced session""" - with self._lock: - if self._session is None or self._should_refresh_session(): - self._session = self._create_enhanced_session() - self._session_created_time = time.time() - self._request_count = 0 - self.logger.debug("Created new enhanced session") - return self._session - - def _should_refresh_session(self) -> bool: - """Check if session should be refreshed""" - session_age = time.time() - self._session_created_time - return (session_age > 600 or # 10 minutes - self._request_count > 50) # 50 requests - + """Get next session from pool (round-robin), refreshing stale sessions.""" + with self._session_pool_lock: + if not self._session_pool: + self._init_session_pool() + idx = self._session_index % len(self._session_pool) + self._session_index += 1 + return self._session_pool[idx] + + def _rotate_session_on_ratelimit(self): + """Replace the current session slot with a fresh one after a rate limit.""" + with self._session_pool_lock: + profile = random.choice(_BROWSER_PROFILES) + idx = self._session_index % len(self._session_pool) + self._session_pool[idx] = self._create_session_from_profile(profile) + def _rate_limit_delay(self): - """Apply intelligent rate limiting""" - with self._lock: - current_time = time.time() - time_since_last = current_time - self._last_request_time - - # Minimum delay based on request frequency - min_delay = 0.1 if self._request_count < 10 else 0.2 - - if time_since_last < min_delay: - delay = min_delay - time_since_last - if self.config.jitter: - delay += random.uniform(0, delay * 0.5) - time.sleep(delay) - - self._last_request_time = time.time() - self._request_count += 1 - - def _retry_with_backoff(self, func, *args, **kwargs): - """Execute function with exponential backoff retry""" + """Delegate to module-level shared rate limiter.""" + _global_rate_limit_delay() + + def _retry_with_backoff(self, func, *args, on_rate_limit_cb=None, **kwargs): + """Execute function with exponential backoff retry and adaptive throttling. + + Args: + func: callable to execute + on_rate_limit_cb: optional zero-arg callback called when a rate limit + is detected (e.g. to refresh the caller's yf.Ticker) + """ last_exception = None - + for attempt in range(self.config.max_retries + 1): try: self._rate_limit_delay() - return func(*args, **kwargs) - + result = func(*args, **kwargs) + _global_on_success() + return result + except Exception as e: last_exception = e error_str = str(e).lower() - + # Check if it's a rate limit or 401 error - if ("rate limit" in error_str or "429" in error_str or + if ("rate limit" in error_str or "429" in error_str or "401" in error_str or "unauthorized" in error_str): - + + _global_on_rate_limit() + if attempt < self.config.max_retries: delay = min( self.config.base_delay * (2 ** attempt), @@ -358,22 +450,21 @@ class EnhancedYFinance: ) if self.config.jitter: delay += random.uniform(0, delay * 0.3) - + if "401" in error_str: - self.logger.debug(f"401 error, refreshing session and retrying in {delay:.2f}s (attempt {attempt + 1})") + self.logger.debug(f"401 error, rotating session and retrying in {delay:.2f}s (attempt {attempt + 1})") else: - self.logger.warning(f"Rate limit hit, retrying in {delay:.2f}s (attempt {attempt + 1})") - + self.logger.warning(f"Rate limit hit, rotating session and retrying in {delay:.2f}s (attempt {attempt + 1})") + time.sleep(delay) - - # Refresh session on auth/rate limit error - with self._lock: - self._session = None + self._rotate_session_on_ratelimit() + if on_rate_limit_cb is not None: + on_rate_limit_cb() continue else: # Non-rate-limit error, re-raise immediately raise e - + # All retries exhausted raise last_exception @@ -397,13 +488,21 @@ class EnhancedTicker: def __init__(self, symbol: str, enhanced_yf: EnhancedYFinance): self.symbol = symbol.upper() self.enhanced_yf = enhanced_yf - self._yf_ticker = None - + # Each EnhancedTicker gets a dedicated session from the pool at creation + # time, ensuring no session is shared between concurrent threads. + self._dedicated_session = enhanced_yf.session + self._yf_ticker = yf.Ticker(self.symbol, session=self._dedicated_session) + + def _refresh_yf_ticker(self): + """Get a fresh session from the pool and rebuild the underlying yf.Ticker. + Called after a rate-limit event to rotate to a different browser fingerprint. + """ + self._dedicated_session = self.enhanced_yf.session + self._yf_ticker = yf.Ticker(self.symbol, session=self._dedicated_session) + @property def yf_ticker(self): - """Get yfinance ticker with enhanced session""" - if self._yf_ticker is None: - self._yf_ticker = yf.Ticker(self.symbol, session=self.enhanced_yf.session) + """Return the cached yf.Ticker (thread-safe: each instance has its own session).""" return self._yf_ticker def __getattr__(self, name): @@ -451,8 +550,10 @@ class EnhancedTicker: start=start, end=end, **kwargs ) - data = self.enhanced_yf._retry_with_backoff(_get_history) - + data = self.enhanced_yf._retry_with_backoff( + _get_history, on_rate_limit_cb=self._refresh_yf_ticker + ) + # Cache the data if successful and caching is enabled if (self.enhanced_yf._cache and data is not None and not data.empty and @@ -503,20 +604,13 @@ class EnhancedTicker: self.enhanced_yf._cache.clear() -def download(tickers: Union[str, list], - period: str = "1mo", +def download(tickers: Union[str, list], + period: str = "1mo", interval: str = "1d", **kwargs) -> pd.DataFrame: """Enhanced download function with better rate limiting""" - - config = RequestConfig( - max_retries=5, - base_delay=1.0, - max_delay=120.0 - ) - - enhanced_yf = EnhancedYFinance(config) - + enhanced_yf = _get_global_enhanced_yf() + def _download(): return yf.download( tickers=tickers, @@ -525,21 +619,41 @@ def download(tickers: Union[str, list], session=enhanced_yf.session, **kwargs ) - + return enhanced_yf._retry_with_backoff(_download) # Global configuration _global_config = RequestConfig() +# Module-level singleton EnhancedYFinance — shared by all Ticker() calls so that +# _global_rate_limit_delay() and the session pool are coordinated across all +# concurrent requests (e.g., multiple FastAPI handler coroutines running in +# executor threads at the same time). +_global_enhanced_yf: Optional['EnhancedYFinance'] = None +_global_enhanced_yf_lock = threading.Lock() + + +def _get_global_enhanced_yf() -> 'EnhancedYFinance': + global _global_enhanced_yf + if _global_enhanced_yf is None: + with _global_enhanced_yf_lock: + if _global_enhanced_yf is None: + _global_enhanced_yf = EnhancedYFinance(_global_config) + return _global_enhanced_yf + + def set_config(**kwargs): """Set global configuration for enhanced yfinance""" - global _global_config + global _global_config, _global_enhanced_yf for key, value in kwargs.items(): if hasattr(_global_config, key): setattr(_global_config, key, value) else: raise ValueError(f"Unknown configuration option: {key}") + # Reset singleton so next Ticker() call picks up new config + with _global_enhanced_yf_lock: + _global_enhanced_yf = None def get_config() -> RequestConfig: """Get current global configuration""" @@ -547,24 +661,20 @@ def get_config() -> RequestConfig: # Convenience functions with full yfinance API compatibility def Ticker(symbol: str, session=None, proxy=None) -> EnhancedTicker: - """Create enhanced ticker (fully compatible with yfinance.Ticker)""" - config = RequestConfig( - max_retries=_global_config.max_retries, - base_delay=_global_config.base_delay, - max_delay=_global_config.max_delay, - jitter=_global_config.jitter, - user_agents=_global_config.user_agents, - enable_cache=_global_config.enable_cache, - cache_dir=_global_config.cache_dir - ) - - enhanced_yf = EnhancedYFinance(config) - - # Override session if provided (for compatibility) + """Create enhanced ticker (fully compatible with yfinance.Ticker). + + All calls share the module-level EnhancedYFinance singleton so the + adaptive rate limiter and session pool are coordinated process-wide. + """ + enhanced_yf = _get_global_enhanced_yf() + ticker = enhanced_yf.get_ticker(symbol) + + # Override session if caller explicitly provides one (rare compatibility path) if session is not None: - enhanced_yf._session = session - - return enhanced_yf.get_ticker(symbol) + with enhanced_yf._session_pool_lock: + enhanced_yf._session_pool = [session] + + return ticker class EnhancedTickers: """Enhanced Tickers class with full yfinance compatibility"""