You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
304 lines
12 KiB
Python
304 lines
12 KiB
Python
"""
|
|
Yahoo Finance 52-Week Gainers Service
|
|
야후 파이낸스에서 52주 상승률 상위 주식 데이터를 가져오는 서비스
|
|
"""
|
|
|
|
import logging
|
|
import re
|
|
import asyncio
|
|
import time
|
|
from typing import List, Dict, Optional
|
|
from datetime import datetime
|
|
import random
|
|
|
|
from curl_cffi import requests
|
|
from bs4 import BeautifulSoup
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class Yahoo52WeekGainersService:
|
|
"""Yahoo Finance 52-Week Gainers 서비스 with intelligent rate limiting"""
|
|
|
|
def __init__(self):
|
|
self.base_url = "https://finance.yahoo.com/markets/stocks/52-week-gainers/"
|
|
self.session = None
|
|
self._last_session_time = None
|
|
self._request_count = 0
|
|
self._last_request_time = 0
|
|
|
|
# Rate limiting 설정
|
|
self.min_delay = 1.0 # 최소 1초 대기
|
|
self.max_delay = 3.0 # 최대 3초 대기
|
|
self.batch_delay = 5.0 # 배치 간 대기시간
|
|
self.requests_per_batch = 3 # 배치당 요청 수
|
|
|
|
def _get_session(self) -> requests.Session:
|
|
"""세션 생성 및 관리"""
|
|
current_time = datetime.now()
|
|
|
|
# 세션이 없거나 5분 이상 지났으면 새로 생성 (더 자주 갱신)
|
|
if (self.session is None or
|
|
self._last_session_time is None or
|
|
(current_time - self._last_session_time).seconds > 300):
|
|
|
|
self.session = requests.Session(impersonate='chrome')
|
|
self._last_session_time = current_time
|
|
self._request_count = 0 # 리셋
|
|
logger.info("🔄 Created new Yahoo Finance session for 52-week gainers")
|
|
|
|
return self.session
|
|
|
|
async def _smart_delay(self, page_num: int = 0):
|
|
"""Intelligent rate limiting with progressive delays"""
|
|
current_time = time.time()
|
|
|
|
# 기본 지연시간 (랜덤)
|
|
base_delay = random.uniform(self.min_delay, self.max_delay)
|
|
|
|
# 배치 처리 지연
|
|
if self._request_count > 0 and self._request_count % self.requests_per_batch == 0:
|
|
batch_delay = self.batch_delay + random.uniform(0, 2.0)
|
|
logger.info(f"🛑 Batch delay: {batch_delay:.1f}s (after {self._request_count} requests)")
|
|
await asyncio.sleep(batch_delay)
|
|
else:
|
|
await asyncio.sleep(base_delay)
|
|
|
|
# 요청 간 최소 간격 보장
|
|
time_since_last = current_time - self._last_request_time
|
|
if time_since_last < self.min_delay:
|
|
additional_delay = self.min_delay - time_since_last
|
|
await asyncio.sleep(additional_delay)
|
|
|
|
self._last_request_time = time.time()
|
|
self._request_count += 1
|
|
|
|
# 페이지 번호에 따른 progressive delay
|
|
if page_num > 3:
|
|
extra_delay = (page_num - 3) * 0.5 # 4페이지부터 0.5초씩 추가
|
|
logger.debug(f"⏱️ Progressive delay for page {page_num}: +{extra_delay:.1f}s")
|
|
await asyncio.sleep(extra_delay)
|
|
|
|
async def get_total_count(self) -> int:
|
|
"""총 주식 개수 확인"""
|
|
session = self._get_session()
|
|
url = f'{self.base_url}?start=0&count=200'
|
|
|
|
try:
|
|
await self._smart_delay()
|
|
response = session.get(url, timeout=30)
|
|
|
|
if response.status_code == 200:
|
|
soup = BeautifulSoup(response.text, 'html.parser')
|
|
|
|
# 페이지네이션 정보 찾기
|
|
pagination_texts = soup.find_all(string=re.compile(r'\d+-\d+ of \d+'))
|
|
|
|
for text in pagination_texts:
|
|
match = re.search(r'(\d+)-(\d+) of (\d+)', text.strip())
|
|
if match:
|
|
total = int(match.group(3))
|
|
logger.info(f"📊 Total 52-week gainers available: {total}")
|
|
return total
|
|
except Exception as e:
|
|
logger.error(f"Error getting total count: {e}")
|
|
|
|
return 1350 # 기본값
|
|
|
|
async def get_page_stocks(self, start: int, count: int, page_num: int = 0) -> List[Dict]:
|
|
"""
|
|
특정 페이지의 52주 상승 주식 데이터를 가져옴
|
|
|
|
Args:
|
|
start: 시작 인덱스 (0, 200, 400, ...)
|
|
count: 가져올 주식 개수 (최대 200)
|
|
page_num: 페이지 번호 (rate limiting 용)
|
|
|
|
Returns:
|
|
주식 정보 리스트
|
|
"""
|
|
session = self._get_session()
|
|
url = f'{self.base_url}?start={start}&count={count}'
|
|
|
|
try:
|
|
await self._smart_delay(page_num)
|
|
|
|
logger.debug(f'📡 Fetching page {page_num + 1} (start={start})')
|
|
response = session.get(url, timeout=30)
|
|
|
|
if response.status_code != 200:
|
|
logger.warning(f'⚠️ HTTP {response.status_code} for start={start}')
|
|
return []
|
|
|
|
soup = BeautifulSoup(response.text, 'html.parser')
|
|
|
|
# 메인 테이블 찾기
|
|
table = soup.find('table')
|
|
if not table:
|
|
logger.warning('⚠️ No table found')
|
|
return []
|
|
|
|
tbody = table.find('tbody')
|
|
if not tbody:
|
|
logger.warning('⚠️ No tbody found')
|
|
return []
|
|
|
|
rows = tbody.find_all('tr')
|
|
logger.debug(f'📊 Found {len(rows)} data rows')
|
|
|
|
stocks = []
|
|
for row in rows:
|
|
try:
|
|
cells = row.find_all('td')
|
|
if len(cells) < 8: # 최소 8개 컬럼 필요
|
|
continue
|
|
|
|
# 데이터 추출 (most-active와 동일한 구조)
|
|
symbol_cell = cells[0]
|
|
symbol_link = symbol_cell.find('a')
|
|
if not symbol_link:
|
|
symbol = symbol_cell.text.strip()
|
|
else:
|
|
symbol = symbol_link.text.strip()
|
|
|
|
# 회사명
|
|
company_name = cells[1].text.strip() if len(cells) > 1 else "N/A"
|
|
|
|
# 가격 및 변화 정보
|
|
price_change_raw = cells[3].text.strip() if len(cells) > 3 else "N/A"
|
|
change_amount = cells[4].text.strip() if len(cells) > 4 else "N/A"
|
|
change_percent = cells[5].text.strip() if len(cells) > 5 else "N/A"
|
|
volume = cells[6].text.strip() if len(cells) > 6 else "N/A"
|
|
avg_volume = cells[7].text.strip() if len(cells) > 7 else "N/A"
|
|
|
|
# 52주 최고가 (8번째 컬럼이 있다면)
|
|
high_52w = cells[8].text.strip() if len(cells) > 8 else "N/A"
|
|
|
|
# 현재가 추출
|
|
current_price = "N/A"
|
|
if price_change_raw and price_change_raw != "N/A":
|
|
parts = price_change_raw.split()
|
|
if parts:
|
|
current_price = parts[0]
|
|
|
|
stock_data = {
|
|
'symbol': symbol,
|
|
'company_name': company_name,
|
|
'current_price': current_price,
|
|
'price_change_raw': price_change_raw,
|
|
'change_amount': change_amount,
|
|
'change_percent': change_percent,
|
|
'volume': volume,
|
|
'avg_volume': avg_volume,
|
|
'high_52w': high_52w,
|
|
'scraped_at': datetime.now().isoformat()
|
|
}
|
|
|
|
stocks.append(stock_data)
|
|
|
|
except Exception as e:
|
|
logger.debug(f'⚠️ Error processing row: {e}')
|
|
continue
|
|
|
|
logger.info(f'✅ Page {page_num + 1}: Extracted {len(stocks)} stocks from start={start}')
|
|
return stocks
|
|
|
|
except Exception as e:
|
|
logger.error(f'❌ Error fetching page data (start={start}): {e}')
|
|
# Rate limit에 걸렸을 가능성이 있으면 추가 대기
|
|
if 'rate' in str(e).lower() or 'limit' in str(e).lower():
|
|
logger.warning(f'🚫 Possible rate limiting detected, adding extra delay')
|
|
await asyncio.sleep(10.0)
|
|
return []
|
|
|
|
async def get_all_52week_gainers(self, limit: Optional[int] = None, max_pages: Optional[int] = None) -> Dict:
|
|
"""
|
|
모든 페이지에서 52주 상승 주식 데이터를 가져옴
|
|
|
|
Args:
|
|
limit: 최대 반환할 주식 개수 (None이면 전체)
|
|
max_pages: 최대 페이지 수 제한 (rate limiting 방지)
|
|
|
|
Returns:
|
|
결과 딕셔너리
|
|
"""
|
|
try:
|
|
logger.info('🔍 Starting Yahoo Finance 52-week gainers collection')
|
|
start_time = time.time()
|
|
|
|
total_count = await self.get_total_count()
|
|
|
|
all_stocks = []
|
|
page_size = 200
|
|
pages_needed = (total_count + page_size - 1) // page_size
|
|
|
|
# max_pages 제한 적용
|
|
if max_pages:
|
|
pages_needed = min(pages_needed, max_pages)
|
|
logger.info(f'📄 Limited to {max_pages} pages (of {(total_count + page_size - 1) // page_size} total)')
|
|
|
|
logger.info(f'📄 Fetching {pages_needed} pages for up to {total_count} total stocks')
|
|
|
|
for page in range(pages_needed):
|
|
start = page * page_size
|
|
logger.info(f' 📄 Processing page {page + 1}/{pages_needed} (start={start})')
|
|
|
|
page_stocks = await self.get_page_stocks(start, page_size, page)
|
|
if page_stocks:
|
|
all_stocks.extend(page_stocks)
|
|
logger.info(f' ✅ Added {len(page_stocks)} stocks (total: {len(all_stocks)})')
|
|
else:
|
|
logger.warning(f' ⚠️ No data from page {page + 1}')
|
|
|
|
# limit 적용
|
|
if limit and len(all_stocks) >= limit:
|
|
all_stocks = all_stocks[:limit]
|
|
logger.info(f'🎯 Reached limit of {limit} stocks, stopping')
|
|
break
|
|
|
|
elapsed_time = time.time() - start_time
|
|
logger.info(f'🎯 Final result: {len(all_stocks)} stocks collected in {elapsed_time:.1f}s')
|
|
|
|
return {
|
|
'success': True,
|
|
'data': {
|
|
'stocks': all_stocks,
|
|
'total_available': total_count,
|
|
'returned_count': len(all_stocks),
|
|
'pages_fetched': min(page + 1, pages_needed) if 'page' in locals() else pages_needed,
|
|
'scraped_at': datetime.now().isoformat(),
|
|
'elapsed_time_seconds': round(elapsed_time, 1)
|
|
},
|
|
'metadata': {
|
|
'source': 'finance.yahoo.com',
|
|
'endpoint': 'markets/stocks/52-week-gainers',
|
|
'method': 'intelligent_web_scraping',
|
|
'rate_limit_bypass': 'curl_cffi_chrome_impersonation_with_smart_delays',
|
|
'requests_made': self._request_count
|
|
}
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.error(f'❌ Error in get_all_52week_gainers: {e}')
|
|
return {
|
|
'success': False,
|
|
'error': str(e),
|
|
'data': None
|
|
}
|
|
|
|
async def get_52week_gainers(self, limit: int = 200, max_pages: int = 3) -> Dict:
|
|
"""
|
|
상위 N개의 52주 상승 주식 데이터를 가져옴 (기본적으로 안전한 제한)
|
|
|
|
Args:
|
|
limit: 반환할 주식 개수
|
|
max_pages: 최대 페이지 수 (기본 3페이지 = 600개 주식)
|
|
|
|
Returns:
|
|
결과 딕셔너리
|
|
"""
|
|
return await self.get_all_52week_gainers(limit=limit, max_pages=max_pages)
|
|
|
|
|
|
# 싱글톤 인스턴스
|
|
yahoo_52week_gainers_service = Yahoo52WeekGainersService() |