stock-oracle/app/services/yahoo_most_active_service.py

"""
Yahoo Finance Most Active Stocks Service
야후 파이낸스에서 가장 활발한 주식 데이터를 가져오는 서비스
"""

import logging
import re
from typing import List, Dict, Optional
from datetime import datetime

from curl_cffi import requests
from bs4 import BeautifulSoup

logger = logging.getLogger(__name__)


class YahooMostActiveService:
    """Yahoo Finance Most Active Stocks 서비스"""
    
    def __init__(self):
        self.base_url = "https://finance.yahoo.com/markets/stocks/most-active/"
        self.session = None
        self._last_session_time = None
        
    def _get_session(self) -> requests.Session:
        """세션 생성 및 관리"""
        current_time = datetime.now()
        
        # 세션이 없거나 10분 이상 지났으면 새로 생성
        if (self.session is None or 
            self._last_session_time is None or 
            (current_time - self._last_session_time).seconds > 600):
            
            self.session = requests.Session(impersonate='chrome')
            self._last_session_time = current_time
            logger.info("🔄 Created new Yahoo Finance session")
        
        return self.session
    
    def get_total_count(self) -> int:
        """총 주식 개수 확인"""
        session = self._get_session()
        url = f'{self.base_url}?start=0&count=100'
        
        try:
            response = session.get(url, timeout=30)
            if response.status_code == 200:
                soup = BeautifulSoup(response.text, 'html.parser')
                
                # 페이지네이션 정보 찾기
                pagination_texts = soup.find_all(string=re.compile(r'\d+-\d+ of \d+'))
                
                for text in pagination_texts:
                    match = re.search(r'(\d+)-(\d+) of (\d+)', text.strip())
                    if match:
                        total = int(match.group(3))
                        logger.info(f"📊 Total active stocks available: {total}")
                        return total
        except Exception as e:
            logger.error(f"Error getting total count: {e}")
        
        return 168  # 기본값
    
    def get_page_stocks(self, start: int, count: int) -> List[Dict]:
        """
        특정 페이지의 주식 데이터를 가져옴
        
        Args:
            start: 시작 인덱스 (0, 100, 200, ...)
            count: 가져올 주식 개수 (최대 100)
        
        Returns:
            주식 정보 리스트
        """
        session = self._get_session()
        url = f'{self.base_url}?start={start}&count={count}'
        
        try:
            response = session.get(url, timeout=30)
            logger.debug(f'📡 HTTP Status: {response.status_code} for start={start}')
            
            if response.status_code != 200:
                return []
            
            soup = BeautifulSoup(response.text, 'html.parser')
            
            # 메인 테이블 찾기
            table = soup.find('table')
            if not table:
                logger.warning('⚠️ No table found')
                return []
            
            tbody = table.find('tbody')
            if not tbody:
                logger.warning('⚠️ No tbody found')
                return []
            
            rows = tbody.find_all('tr')
            logger.debug(f'📊 Found {len(rows)} data rows')
            
            stocks = []
            for row in rows:
                try:
                    cells = row.find_all('td')
                    if len(cells) < 6:  # 최소 6개 컬럼 필요
                        continue
                    
                    # 심볼 (첫 번째 컬럼)
                    symbol_cell = cells[0]
                    symbol_link = symbol_cell.find('a')
                    if not symbol_link:
                        symbol = symbol_cell.text.strip()
                    else:
                        symbol = symbol_link.text.strip()
                    
                    # 회사명 (두 번째 컬럼)
                    company_name = cells[1].text.strip() if len(cells) > 1 else "N/A"
                    
                    # 가격 및 변화 정보 (네 번째 컬럼에 모든 가격 정보가 있음)
                    price_change_raw = cells[3].text.strip() if len(cells) > 3 else "N/A"
                    change_raw = cells[4].text.strip() if len(cells) > 4 else "N/A" 
                    change_pct_raw = cells[5].text.strip() if len(cells) > 5 else "N/A"
                    volume_raw = cells[6].text.strip() if len(cells) > 6 else "N/A"
                    avg_volume_raw = cells[7].text.strip() if len(cells) > 7 else "N/A"
                    
                    # 가격 정보 파싱 (예: "181.96 +0.42 (+0.23%)")
                    current_price = "N/A"
                    if price_change_raw and price_change_raw != "N/A":
                        parts = price_change_raw.split()
                        if parts:
                            current_price = parts[0]
                    
                    # 구조화된 데이터 생성
                    stock_data = {
                        'symbol': symbol,
                        'company_name': company_name,
                        'current_price': current_price,
                        'price_change_raw': price_change_raw,
                        'change_amount': change_raw,
                        'change_percent': change_pct_raw,
                        'volume': volume_raw,
                        'avg_volume': avg_volume_raw,
                        'scraped_at': datetime.now().isoformat()
                    }
                    
                    stocks.append(stock_data)
                    
                except Exception as e:
                    logger.debug(f'⚠️ Error processing row: {e}')
                    continue
            
            logger.info(f'✅ Successfully extracted {len(stocks)} stocks from start={start}')
            return stocks
            
        except Exception as e:
            logger.error(f'❌ Error fetching page data (start={start}): {e}')
            return []
    
    async def get_all_most_active_stocks(self, limit: Optional[int] = None) -> Dict:
        """
        모든 페이지에서 활발한 주식 데이터를 가져옴
        
        Args:
            limit: 최대 반환할 주식 개수 (None이면 전체)
            
        Returns:
            결과 딕셔너리
        """
        try:
            logger.info('🔍 Starting Yahoo Finance most active stocks collection')
            
            total_count = self.get_total_count()
            
            all_stocks = []
            page_size = 100
            pages_needed = (total_count + page_size - 1) // page_size
            
            logger.info(f'📄 Fetching {pages_needed} pages for {total_count} total stocks')
            
            for page in range(pages_needed):
                start = page * page_size
                logger.debug(f'   Page {page + 1}/{pages_needed} (start={start})')
                
                page_stocks = self.get_page_stocks(start, page_size)
                if page_stocks:
                    all_stocks.extend(page_stocks)
                    logger.debug(f'   ✅ Added {len(page_stocks)} stocks (total: {len(all_stocks)})')
                else:
                    logger.warning(f'   ⚠️ No data from page {page + 1}')
                
                # limit 적용
                if limit and len(all_stocks) >= limit:
                    all_stocks = all_stocks[:limit]
                    break
            
            logger.info(f'🎯 Final result: {len(all_stocks)} stocks collected')
            
            return {
                'success': True,
                'data': {
                    'stocks': all_stocks,
                    'total_available': total_count,
                    'returned_count': len(all_stocks),
                    'pages_fetched': min(page + 1, pages_needed) if 'page' in locals() else pages_needed,
                    'scraped_at': datetime.now().isoformat()
                },
                'metadata': {
                    'source': 'finance.yahoo.com',
                    'endpoint': 'markets/stocks/most-active',
                    'method': 'web_scraping',
                    'rate_limit_bypass': 'curl_cffi_chrome_impersonation'
                }
            }
            
        except Exception as e:
            logger.error(f'❌ Error in get_all_most_active_stocks: {e}')
            return {
                'success': False,
                'error': str(e),
                'data': None
            }
    
    async def get_most_active_stocks(self, limit: int = 100) -> Dict:
        """
        상위 N개의 활발한 주식 데이터를 가져옴
        
        Args:
            limit: 반환할 주식 개수
            
        Returns:
            결과 딕셔너리
        """
        return await self.get_all_most_active_stocks(limit=limit)


# 싱글톤 인스턴스
yahoo_most_active_service = YahooMostActiveService()