""" Yahoo Finance Most Active Stocks Service 야후 파이낸스에서 가장 활발한 주식 데이터를 가져오는 서비스 """ import logging import re from typing import List, Dict, Optional from datetime import datetime from curl_cffi import requests from bs4 import BeautifulSoup logger = logging.getLogger(__name__) class YahooMostActiveService: """Yahoo Finance Most Active Stocks 서비스""" def __init__(self): self.base_url = "https://finance.yahoo.com/markets/stocks/most-active/" self.session = None self._last_session_time = None def _get_session(self) -> requests.Session: """세션 생성 및 관리""" current_time = datetime.now() # 세션이 없거나 10분 이상 지났으면 새로 생성 if (self.session is None or self._last_session_time is None or (current_time - self._last_session_time).seconds > 600): self.session = requests.Session(impersonate='chrome') self._last_session_time = current_time logger.info("🔄 Created new Yahoo Finance session") return self.session def get_total_count(self) -> int: """총 주식 개수 확인""" session = self._get_session() url = f'{self.base_url}?start=0&count=100' try: response = session.get(url, timeout=30) if response.status_code == 200: soup = BeautifulSoup(response.text, 'html.parser') # 페이지네이션 정보 찾기 pagination_texts = soup.find_all(string=re.compile(r'\d+-\d+ of \d+')) for text in pagination_texts: match = re.search(r'(\d+)-(\d+) of (\d+)', text.strip()) if match: total = int(match.group(3)) logger.info(f"📊 Total active stocks available: {total}") return total except Exception as e: logger.error(f"Error getting total count: {e}") return 168 # 기본값 def get_page_stocks(self, start: int, count: int) -> List[Dict]: """ 특정 페이지의 주식 데이터를 가져옴 Args: start: 시작 인덱스 (0, 100, 200, ...) count: 가져올 주식 개수 (최대 100) Returns: 주식 정보 리스트 """ session = self._get_session() url = f'{self.base_url}?start={start}&count={count}' try: response = session.get(url, timeout=30) logger.debug(f'📡 HTTP Status: {response.status_code} for start={start}') if response.status_code != 200: return [] soup = BeautifulSoup(response.text, 'html.parser') # 메인 테이블 찾기 table = soup.find('table') if not table: logger.warning('⚠️ No table found') return [] tbody = table.find('tbody') if not tbody: logger.warning('⚠️ No tbody found') return [] rows = tbody.find_all('tr') logger.debug(f'📊 Found {len(rows)} data rows') stocks = [] for row in rows: try: cells = row.find_all('td') if len(cells) < 6: # 최소 6개 컬럼 필요 continue # 심볼 (첫 번째 컬럼) symbol_cell = cells[0] symbol_link = symbol_cell.find('a') if not symbol_link: symbol = symbol_cell.text.strip() else: symbol = symbol_link.text.strip() # 회사명 (두 번째 컬럼) company_name = cells[1].text.strip() if len(cells) > 1 else "N/A" # 가격 및 변화 정보 (네 번째 컬럼에 모든 가격 정보가 있음) price_change_raw = cells[3].text.strip() if len(cells) > 3 else "N/A" change_raw = cells[4].text.strip() if len(cells) > 4 else "N/A" change_pct_raw = cells[5].text.strip() if len(cells) > 5 else "N/A" volume_raw = cells[6].text.strip() if len(cells) > 6 else "N/A" avg_volume_raw = cells[7].text.strip() if len(cells) > 7 else "N/A" # 가격 정보 파싱 (예: "181.96 +0.42 (+0.23%)") current_price = "N/A" if price_change_raw and price_change_raw != "N/A": parts = price_change_raw.split() if parts: current_price = parts[0] # 구조화된 데이터 생성 stock_data = { 'symbol': symbol, 'company_name': company_name, 'current_price': current_price, 'price_change_raw': price_change_raw, 'change_amount': change_raw, 'change_percent': change_pct_raw, 'volume': volume_raw, 'avg_volume': avg_volume_raw, 'scraped_at': datetime.now().isoformat() } stocks.append(stock_data) except Exception as e: logger.debug(f'⚠️ Error processing row: {e}') continue logger.info(f'✅ Successfully extracted {len(stocks)} stocks from start={start}') return stocks except Exception as e: logger.error(f'❌ Error fetching page data (start={start}): {e}') return [] async def get_all_most_active_stocks(self, limit: Optional[int] = None) -> Dict: """ 모든 페이지에서 활발한 주식 데이터를 가져옴 Args: limit: 최대 반환할 주식 개수 (None이면 전체) Returns: 결과 딕셔너리 """ try: logger.info('🔍 Starting Yahoo Finance most active stocks collection') total_count = self.get_total_count() all_stocks = [] page_size = 100 pages_needed = (total_count + page_size - 1) // page_size logger.info(f'📄 Fetching {pages_needed} pages for {total_count} total stocks') for page in range(pages_needed): start = page * page_size logger.debug(f' Page {page + 1}/{pages_needed} (start={start})') page_stocks = self.get_page_stocks(start, page_size) if page_stocks: all_stocks.extend(page_stocks) logger.debug(f' ✅ Added {len(page_stocks)} stocks (total: {len(all_stocks)})') else: logger.warning(f' ⚠️ No data from page {page + 1}') # limit 적용 if limit and len(all_stocks) >= limit: all_stocks = all_stocks[:limit] break logger.info(f'🎯 Final result: {len(all_stocks)} stocks collected') return { 'success': True, 'data': { 'stocks': all_stocks, 'total_available': total_count, 'returned_count': len(all_stocks), 'pages_fetched': min(page + 1, pages_needed) if 'page' in locals() else pages_needed, 'scraped_at': datetime.now().isoformat() }, 'metadata': { 'source': 'finance.yahoo.com', 'endpoint': 'markets/stocks/most-active', 'method': 'web_scraping', 'rate_limit_bypass': 'curl_cffi_chrome_impersonation' } } except Exception as e: logger.error(f'❌ Error in get_all_most_active_stocks: {e}') return { 'success': False, 'error': str(e), 'data': None } async def get_most_active_stocks(self, limit: int = 100) -> Dict: """ 상위 N개의 활발한 주식 데이터를 가져옴 Args: limit: 반환할 주식 개수 Returns: 결과 딕셔너리 """ return await self.get_all_most_active_stocks(limit=limit) # 싱글톤 인스턴스 yahoo_most_active_service = YahooMostActiveService()