""" Yahoo Finance 52-Week Gainers Service 야후 파이낸스에서 52주 상승률 상위 주식 데이터를 가져오는 서비스 """ import logging import re import asyncio import time from typing import List, Dict, Optional from datetime import datetime import random from curl_cffi import requests from bs4 import BeautifulSoup logger = logging.getLogger(__name__) class Yahoo52WeekGainersService: """Yahoo Finance 52-Week Gainers 서비스 with intelligent rate limiting""" def __init__(self): self.base_url = "https://finance.yahoo.com/markets/stocks/52-week-gainers/" self.session = None self._last_session_time = None self._request_count = 0 self._last_request_time = 0 # Rate limiting 설정 self.min_delay = 1.0 # 최소 1초 대기 self.max_delay = 3.0 # 최대 3초 대기 self.batch_delay = 5.0 # 배치 간 대기시간 self.requests_per_batch = 3 # 배치당 요청 수 def _get_session(self) -> requests.Session: """세션 생성 및 관리""" current_time = datetime.now() # 세션이 없거나 5분 이상 지났으면 새로 생성 (더 자주 갱신) if (self.session is None or self._last_session_time is None or (current_time - self._last_session_time).seconds > 300): self.session = requests.Session(impersonate='chrome') self._last_session_time = current_time self._request_count = 0 # 리셋 logger.info("🔄 Created new Yahoo Finance session for 52-week gainers") return self.session async def _smart_delay(self, page_num: int = 0): """Intelligent rate limiting with progressive delays""" current_time = time.time() # 기본 지연시간 (랜덤) base_delay = random.uniform(self.min_delay, self.max_delay) # 배치 처리 지연 if self._request_count > 0 and self._request_count % self.requests_per_batch == 0: batch_delay = self.batch_delay + random.uniform(0, 2.0) logger.info(f"🛑 Batch delay: {batch_delay:.1f}s (after {self._request_count} requests)") await asyncio.sleep(batch_delay) else: await asyncio.sleep(base_delay) # 요청 간 최소 간격 보장 time_since_last = current_time - self._last_request_time if time_since_last < self.min_delay: additional_delay = self.min_delay - time_since_last await asyncio.sleep(additional_delay) self._last_request_time = time.time() self._request_count += 1 # 페이지 번호에 따른 progressive delay if page_num > 3: extra_delay = (page_num - 3) * 0.5 # 4페이지부터 0.5초씩 추가 logger.debug(f"⏱️ Progressive delay for page {page_num}: +{extra_delay:.1f}s") await asyncio.sleep(extra_delay) async def get_total_count(self) -> int: """총 주식 개수 확인""" session = self._get_session() url = f'{self.base_url}?start=0&count=200' try: await self._smart_delay() response = session.get(url, timeout=30) if response.status_code == 200: soup = BeautifulSoup(response.text, 'html.parser') # 페이지네이션 정보 찾기 pagination_texts = soup.find_all(string=re.compile(r'\d+-\d+ of \d+')) for text in pagination_texts: match = re.search(r'(\d+)-(\d+) of (\d+)', text.strip()) if match: total = int(match.group(3)) logger.info(f"📊 Total 52-week gainers available: {total}") return total except Exception as e: logger.error(f"Error getting total count: {e}") return 1350 # 기본값 async def get_page_stocks(self, start: int, count: int, page_num: int = 0) -> List[Dict]: """ 특정 페이지의 52주 상승 주식 데이터를 가져옴 Args: start: 시작 인덱스 (0, 200, 400, ...) count: 가져올 주식 개수 (최대 200) page_num: 페이지 번호 (rate limiting 용) Returns: 주식 정보 리스트 """ session = self._get_session() url = f'{self.base_url}?start={start}&count={count}' try: await self._smart_delay(page_num) logger.debug(f'📡 Fetching page {page_num + 1} (start={start})') response = session.get(url, timeout=30) if response.status_code != 200: logger.warning(f'⚠️ HTTP {response.status_code} for start={start}') return [] soup = BeautifulSoup(response.text, 'html.parser') # 메인 테이블 찾기 table = soup.find('table') if not table: logger.warning('⚠️ No table found') return [] tbody = table.find('tbody') if not tbody: logger.warning('⚠️ No tbody found') return [] rows = tbody.find_all('tr') logger.debug(f'📊 Found {len(rows)} data rows') stocks = [] for row in rows: try: cells = row.find_all('td') if len(cells) < 8: # 최소 8개 컬럼 필요 continue # 데이터 추출 (most-active와 동일한 구조) symbol_cell = cells[0] symbol_link = symbol_cell.find('a') if not symbol_link: symbol = symbol_cell.text.strip() else: symbol = symbol_link.text.strip() # 회사명 company_name = cells[1].text.strip() if len(cells) > 1 else "N/A" # 가격 및 변화 정보 price_change_raw = cells[3].text.strip() if len(cells) > 3 else "N/A" change_amount = cells[4].text.strip() if len(cells) > 4 else "N/A" change_percent = cells[5].text.strip() if len(cells) > 5 else "N/A" volume = cells[6].text.strip() if len(cells) > 6 else "N/A" avg_volume = cells[7].text.strip() if len(cells) > 7 else "N/A" # 52주 최고가 (8번째 컬럼이 있다면) high_52w = cells[8].text.strip() if len(cells) > 8 else "N/A" # 현재가 추출 current_price = "N/A" if price_change_raw and price_change_raw != "N/A": parts = price_change_raw.split() if parts: current_price = parts[0] stock_data = { 'symbol': symbol, 'company_name': company_name, 'current_price': current_price, 'price_change_raw': price_change_raw, 'change_amount': change_amount, 'change_percent': change_percent, 'volume': volume, 'avg_volume': avg_volume, 'high_52w': high_52w, 'scraped_at': datetime.now().isoformat() } stocks.append(stock_data) except Exception as e: logger.debug(f'⚠️ Error processing row: {e}') continue logger.info(f'✅ Page {page_num + 1}: Extracted {len(stocks)} stocks from start={start}') return stocks except Exception as e: logger.error(f'❌ Error fetching page data (start={start}): {e}') # Rate limit에 걸렸을 가능성이 있으면 추가 대기 if 'rate' in str(e).lower() or 'limit' in str(e).lower(): logger.warning(f'🚫 Possible rate limiting detected, adding extra delay') await asyncio.sleep(10.0) return [] async def get_all_52week_gainers(self, limit: Optional[int] = None, max_pages: Optional[int] = None) -> Dict: """ 모든 페이지에서 52주 상승 주식 데이터를 가져옴 Args: limit: 최대 반환할 주식 개수 (None이면 전체) max_pages: 최대 페이지 수 제한 (rate limiting 방지) Returns: 결과 딕셔너리 """ try: logger.info('🔍 Starting Yahoo Finance 52-week gainers collection') start_time = time.time() total_count = await self.get_total_count() all_stocks = [] page_size = 200 pages_needed = (total_count + page_size - 1) // page_size # max_pages 제한 적용 if max_pages: pages_needed = min(pages_needed, max_pages) logger.info(f'📄 Limited to {max_pages} pages (of {(total_count + page_size - 1) // page_size} total)') logger.info(f'📄 Fetching {pages_needed} pages for up to {total_count} total stocks') for page in range(pages_needed): start = page * page_size logger.info(f' 📄 Processing page {page + 1}/{pages_needed} (start={start})') page_stocks = await self.get_page_stocks(start, page_size, page) if page_stocks: all_stocks.extend(page_stocks) logger.info(f' ✅ Added {len(page_stocks)} stocks (total: {len(all_stocks)})') else: logger.warning(f' ⚠️ No data from page {page + 1}') # limit 적용 if limit and len(all_stocks) >= limit: all_stocks = all_stocks[:limit] logger.info(f'🎯 Reached limit of {limit} stocks, stopping') break elapsed_time = time.time() - start_time logger.info(f'🎯 Final result: {len(all_stocks)} stocks collected in {elapsed_time:.1f}s') return { 'success': True, 'data': { 'stocks': all_stocks, 'total_available': total_count, 'returned_count': len(all_stocks), 'pages_fetched': min(page + 1, pages_needed) if 'page' in locals() else pages_needed, 'scraped_at': datetime.now().isoformat(), 'elapsed_time_seconds': round(elapsed_time, 1) }, 'metadata': { 'source': 'finance.yahoo.com', 'endpoint': 'markets/stocks/52-week-gainers', 'method': 'intelligent_web_scraping', 'rate_limit_bypass': 'curl_cffi_chrome_impersonation_with_smart_delays', 'requests_made': self._request_count } } except Exception as e: logger.error(f'❌ Error in get_all_52week_gainers: {e}') return { 'success': False, 'error': str(e), 'data': None } async def get_52week_gainers(self, limit: int = 200, max_pages: int = 3) -> Dict: """ 상위 N개의 52주 상승 주식 데이터를 가져옴 (기본적으로 안전한 제한) Args: limit: 반환할 주식 개수 max_pages: 최대 페이지 수 (기본 3페이지 = 600개 주식) Returns: 결과 딕셔너리 """ return await self.get_all_52week_gainers(limit=limit, max_pages=max_pages) # 싱글톤 인스턴스 yahoo_52week_gainers_service = Yahoo52WeekGainersService()