You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
505 lines
22 KiB
Python
505 lines
22 KiB
Python
"""
|
|
Stock Market Data endpoints
|
|
주식 시장 데이터 관련 API 엔드포인트
|
|
"""
|
|
|
|
from typing import Optional
|
|
from fastapi import APIRouter, HTTPException, Query, Response
|
|
import logging
|
|
import asyncio
|
|
from datetime import datetime
|
|
|
|
from app.services.yahoo_most_active_service import yahoo_most_active_service
|
|
from app.services.yahoo_52week_gainers_service import yahoo_52week_gainers_service
|
|
from app.utils.cache import build_cache_key, get_cached_response, set_cached_response
|
|
from app.core.config import settings
|
|
|
|
router = APIRouter()
|
|
logger = logging.getLogger("app.api.v1.stocks")
|
|
|
|
|
|
@router.get("/most-active")
|
|
async def get_most_active_stocks(
|
|
response: Response,
|
|
limit: Optional[int] = Query(None, ge=1, le=500, description="Maximum number of stocks to return (1-500). If not specified, returns all available stocks."),
|
|
force_refresh: bool = Query(False, description="If true, bypasses cache and fetches fresh data")
|
|
):
|
|
"""
|
|
Get most actively traded stocks from Yahoo Finance
|
|
|
|
Returns real-time data of the most actively traded stocks including:
|
|
- Stock symbol and company name
|
|
- Current price information
|
|
- Price change and percentage change
|
|
- Trading volume data
|
|
- Average volume data
|
|
|
|
**Data Source**: finance.yahoo.com/markets/stocks/most-active/
|
|
**Update Frequency**: Real-time (scraped on demand)
|
|
**Rate Limiting**: Uses curl_cffi with Chrome impersonation to bypass rate limits
|
|
|
|
**Example Response**:
|
|
```json
|
|
{
|
|
"success": true,
|
|
"data": {
|
|
"stocks": [
|
|
{
|
|
"symbol": "NVDA",
|
|
"company_name": "NVIDIA Corporation",
|
|
"price_raw": "$181.96",
|
|
"change_raw": "+0.42",
|
|
"change_percent_raw": "+0.23%",
|
|
"volume_raw": "45.2M",
|
|
"avg_volume_raw": "42.1M",
|
|
"scraped_at": "2025-01-14T10:30:00"
|
|
}
|
|
],
|
|
"total_available": 168,
|
|
"returned_count": 100,
|
|
"pages_fetched": 1,
|
|
"scraped_at": "2025-01-14T10:30:00"
|
|
},
|
|
"metadata": {
|
|
"source": "finance.yahoo.com",
|
|
"endpoint": "markets/stocks/most-active",
|
|
"method": "web_scraping",
|
|
"rate_limit_bypass": "curl_cffi_chrome_impersonation"
|
|
}
|
|
}
|
|
```
|
|
|
|
**Parameters**:
|
|
- `limit`: Number of stocks to return (optional). If not specified, returns all available stocks (~170)
|
|
|
|
**Notes**:
|
|
- Data is scraped in real-time from Yahoo Finance
|
|
- Without `limit`: Returns all available stocks (typically ~170)
|
|
- With `limit`: Returns top N most active stocks
|
|
- Uses advanced rate limiting bypass techniques (curl_cffi + Chrome impersonation)
|
|
"""
|
|
try:
|
|
# Build cache key by limit parameter
|
|
cache_key = build_cache_key(
|
|
"stocks:most-active",
|
|
f"limit={limit}" if limit is not None else "limit=all"
|
|
)
|
|
|
|
# Try cache (skip if force_refresh)
|
|
if not force_refresh:
|
|
cached = await get_cached_response(cache_key)
|
|
if cached:
|
|
cached_body, etag = cached
|
|
response.headers["X-Cache"] = "HIT"
|
|
response.headers["Cache-Control"] = f"public, max-age={3600}"
|
|
response.headers["ETag"] = etag
|
|
response.headers["X-Data-Source"] = "redis-cache"
|
|
return cached_body
|
|
|
|
# Fetch fresh data
|
|
if limit is None:
|
|
logger.info("📊 Getting ALL most active stocks (no limit specified)")
|
|
result = await yahoo_most_active_service.get_all_most_active_stocks()
|
|
else:
|
|
logger.info(f"📊 Getting most active stocks (limit={limit})")
|
|
result = await yahoo_most_active_service.get_most_active_stocks(limit=limit)
|
|
|
|
if not result['success']:
|
|
logger.error(f"❌ Yahoo Finance service error: {result.get('error')}")
|
|
raise HTTPException(
|
|
status_code=503,
|
|
detail=f"Failed to fetch most active stocks: {result.get('error', 'Unknown error')}"
|
|
)
|
|
|
|
stocks_data = result['data']
|
|
count_msg = f"all {stocks_data['returned_count']}" if limit is None else f"{stocks_data['returned_count']}"
|
|
logger.info(f"✅ Successfully returned {count_msg} most active stocks")
|
|
|
|
response_body = {
|
|
"success": True,
|
|
"message": f"Retrieved {count_msg} most active stocks",
|
|
**result
|
|
}
|
|
|
|
# Set cache after successful fetch
|
|
etag = await set_cached_response(cache_key, response_body, ttl_seconds=3600)
|
|
response.headers["X-Cache"] = "MISS" if not force_refresh else "BYPASS"
|
|
response.headers["Cache-Control"] = f"public, max-age={3600}"
|
|
response.headers["ETag"] = etag
|
|
response.headers["X-Data-Source"] = "scraper"
|
|
|
|
return response_body
|
|
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"❌ Unexpected error in get_most_active_stocks: {e}")
|
|
raise HTTPException(
|
|
status_code=500,
|
|
detail=f"Internal server error while fetching most active stocks: {str(e)}"
|
|
)
|
|
|
|
|
|
@router.get("/52-week-gainers")
|
|
async def get_52week_gainers(
|
|
limit: Optional[int] = Query(None, ge=1, le=1000, description="Maximum number of stocks to return (1-1000). If not specified, returns first 600 stocks (3 pages) for performance."),
|
|
max_pages: Optional[int] = Query(3, ge=1, le=10, description="Maximum pages to fetch (1-10). Each page has ~200 stocks. Higher values may cause rate limiting.")
|
|
):
|
|
"""
|
|
Get 52-week top gaining stocks from Yahoo Finance
|
|
|
|
Returns stocks with highest 52-week price gains including:
|
|
- Stock symbol and company name
|
|
- Current price and 52-week high
|
|
- Price change amount and percentage
|
|
- Trading volume data
|
|
- Gain percentages over 52-week period
|
|
|
|
**Data Source**: finance.yahoo.com/markets/stocks/52-week-gainers/
|
|
**Total Available**: ~1,350 stocks across 7 pages
|
|
**Update Frequency**: Real-time (scraped on demand)
|
|
**Rate Limiting**: Intelligent delays between requests to avoid blocking
|
|
|
|
**Example Response**:
|
|
```json
|
|
{
|
|
"success": true,
|
|
"data": {
|
|
"stocks": [
|
|
{
|
|
"symbol": "EXAMPLE",
|
|
"company_name": "Example Corp",
|
|
"current_price": "10.50",
|
|
"change_percent": "+150.00%",
|
|
"high_52w": "11.00",
|
|
"volume": "1.2M"
|
|
}
|
|
],
|
|
"total_available": 1350,
|
|
"returned_count": 200,
|
|
"elapsed_time_seconds": 15.2
|
|
}
|
|
}
|
|
```
|
|
|
|
**Parameters**:
|
|
- `limit`: Number of stocks to return (optional). Default: returns ~600 stocks (3 pages)
|
|
- `max_pages`: Maximum pages to scrape (1-10). Higher values take longer and may hit rate limits
|
|
|
|
**Performance Notes**:
|
|
- Default (3 pages): ~15-30 seconds, 600 stocks
|
|
- All pages (7 pages): ~45-90 seconds, 1,350 stocks
|
|
- Intelligent rate limiting with progressive delays
|
|
- Session management to avoid detection
|
|
- Automatic retry logic for failed requests
|
|
|
|
**Rate Limiting Strategy**:
|
|
- 1-3 second delays between requests
|
|
- 5+ second delays every 3 requests
|
|
- Progressive delays for later pages
|
|
- Session rotation every 5 minutes
|
|
"""
|
|
try:
|
|
if limit is None:
|
|
logger.info(f"📊 Getting 52-week gainers (default: {max_pages} pages)")
|
|
result = await yahoo_52week_gainers_service.get_52week_gainers(limit=None, max_pages=max_pages)
|
|
else:
|
|
logger.info(f"📊 Getting 52-week gainers (limit={limit}, max_pages={max_pages})")
|
|
result = await yahoo_52week_gainers_service.get_52week_gainers(limit=limit, max_pages=max_pages)
|
|
|
|
if not result['success']:
|
|
logger.error(f"❌ Yahoo Finance 52-week gainers error: {result.get('error')}")
|
|
raise HTTPException(
|
|
status_code=503,
|
|
detail=f"Failed to fetch 52-week gainers: {result.get('error', 'Unknown error')}"
|
|
)
|
|
|
|
stocks_data = result['data']
|
|
count_msg = f"all {stocks_data['returned_count']}" if limit is None else f"{stocks_data['returned_count']}"
|
|
elapsed = stocks_data.get('elapsed_time_seconds', 0)
|
|
|
|
logger.info(f"✅ Successfully returned {count_msg} 52-week gainers in {elapsed}s")
|
|
|
|
return {
|
|
"success": True,
|
|
"message": f"Retrieved {count_msg} 52-week gaining stocks in {elapsed}s",
|
|
**result
|
|
}
|
|
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"❌ Unexpected error in get_52week_gainers: {e}")
|
|
raise HTTPException(
|
|
status_code=500,
|
|
detail=f"Internal server error while fetching 52-week gainers: {str(e)}"
|
|
)
|
|
|
|
|
|
@router.get("/trending")
|
|
async def get_trending_stocks(
|
|
n: Optional[int] = Query(500, ge=1, description="Total number of trending stocks to return after combining most active + gainers (default: 500)"),
|
|
most_active_limit: Optional[int] = Query(None, ge=1, description="Number of most active stocks to include. If not specified, returns all available stocks (~170)."),
|
|
gainers_limit: Optional[int] = Query(None, ge=1, description="Number of 52-week gainers to fetch. If not specified, fetches enough to reach target 'n' after combining with most active.")
|
|
):
|
|
"""
|
|
Get trending stocks combining most active and 52-week gainers
|
|
|
|
Returns a comprehensive list of trending stocks by combining:
|
|
- Most actively traded stocks (high volume, immediate market interest)
|
|
- Top 52-week gainers (strong long-term performance)
|
|
|
|
**Data Sources**:
|
|
- Most Active: finance.yahoo.com/markets/stocks/most-active/
|
|
- 52-Week Gainers: finance.yahoo.com/markets/stocks/52-week-gainers/
|
|
|
|
**Update Frequency**: Real-time (scraped on demand)
|
|
**Rate Limiting**: Optimized parallel fetching with intelligent delays
|
|
|
|
**Example Response**:
|
|
```json
|
|
{
|
|
"success": true,
|
|
"message": "Retrieved 500 trending stocks (170 most active + 330 gainers) in 18.5s",
|
|
"data": {
|
|
"trending_stocks": [
|
|
{
|
|
"symbol": "NVDA",
|
|
"company_name": "NVIDIA Corporation",
|
|
"current_price": "181.96",
|
|
"change_amount": "+0.42",
|
|
"change_percent": "+0.23%",
|
|
"volume": "45.2M",
|
|
"category": "most_active",
|
|
"rank_in_category": 1
|
|
},
|
|
{
|
|
"symbol": "TSLA",
|
|
"company_name": "Tesla Inc",
|
|
"current_price": "248.50",
|
|
"change_amount": "+12.30",
|
|
"change_percent": "+125.50%",
|
|
"volume": "2.1M",
|
|
"high_52w": "250.00",
|
|
"category": "52_week_gainer",
|
|
"rank_in_category": 1
|
|
}
|
|
],
|
|
"summary": {
|
|
"total_stocks": 500,
|
|
"most_active_count": 170,
|
|
"gainers_count": 330,
|
|
"unique_symbols": 485,
|
|
"overlap_count": 15
|
|
},
|
|
"performance": {
|
|
"elapsed_time_seconds": 18.5,
|
|
"most_active_time": 3.1,
|
|
"gainers_time": 15.4,
|
|
"parallel_execution": true
|
|
}
|
|
},
|
|
"metadata": {
|
|
"sources": ["finance.yahoo.com/most-active", "finance.yahoo.com/52-week-gainers"],
|
|
"method": "parallel_scraping_with_intelligent_rate_limiting",
|
|
"categories": ["most_active", "52_week_gainer"]
|
|
}
|
|
}
|
|
```
|
|
|
|
**Parameters**:
|
|
- `n`: Total number of trending stocks to return (default: 500). Final result is limited to this number.
|
|
- `most_active_limit`: Number of most active stocks to include (default: all available ~170 stocks)
|
|
- `gainers_limit`: Number of 52-week gainers to fetch (default: calculated to reach target `n`)
|
|
|
|
**Parameter Coordination**:
|
|
- **Default Behavior**: `n=500`, fetches all most active (~170) + calculates gainers needed (~330)
|
|
- **Custom Total**: Set `n` to control final result size, other parameters auto-adjust
|
|
- **Custom Mix**: Specify `most_active_limit` and/or `gainers_limit` for precise control
|
|
- **Priority**: most_active stocks prioritized, then gainers by rank when limiting to `n`
|
|
|
|
**Performance Notes**:
|
|
- **Default Mode** (n=500): ~15-30 seconds for 500 trending stocks
|
|
- **Fast Mode** (n=200): ~5-10 seconds for 200 trending stocks
|
|
- **Comprehensive Mode** (n=1000+): ~30-60 seconds for large datasets
|
|
- Automatic pagination based on calculated gainers_limit (approximately 200 stocks per page)
|
|
- Parallel execution for optimal performance
|
|
- Smart deduplication to handle overlapping stocks
|
|
|
|
**Categories**:
|
|
- `most_active`: High trading volume, immediate market attention
|
|
- `52_week_gainer`: Strong long-term price performance
|
|
- Stocks may appear in both categories (indicated by overlap_count)
|
|
"""
|
|
try:
|
|
start_time = datetime.now()
|
|
|
|
# Parameter coordination logic
|
|
# 1. If most_active_limit is None, we'll get all available (~170)
|
|
expected_most_active = most_active_limit if most_active_limit is not None else 170
|
|
|
|
# 2. Calculate gainers_limit if not specified to reach target n
|
|
if gainers_limit is None:
|
|
# Calculate how many gainers we need to reach target n
|
|
target_gainers = max(50, n - expected_most_active) # At least 50 gainers
|
|
else:
|
|
target_gainers = gainers_limit
|
|
|
|
# 3. Calculate pages needed for gainers (approximately 200 stocks per page)
|
|
gainers_pages = min(max(1, (target_gainers + 199) // 200), 7) # Ceiling division, max 7 pages
|
|
|
|
logger.info(f"🔥 Getting trending stocks (n={n}, most_active={most_active_limit or 'all'}, target_gainers={target_gainers}, auto_pages={gainers_pages})")
|
|
|
|
# Parallel execution for better performance
|
|
most_active_task = yahoo_most_active_service.get_most_active_stocks(limit=most_active_limit)
|
|
gainers_task = yahoo_52week_gainers_service.get_52week_gainers(limit=target_gainers, max_pages=gainers_pages)
|
|
|
|
# Wait for both tasks to complete
|
|
most_active_result, gainers_result = await asyncio.gather(most_active_task, gainers_task)
|
|
|
|
# Check for errors
|
|
if not most_active_result['success']:
|
|
logger.error(f"❌ Most active stocks error: {most_active_result.get('error')}")
|
|
raise HTTPException(
|
|
status_code=503,
|
|
detail=f"Failed to fetch most active stocks: {most_active_result.get('error', 'Unknown error')}"
|
|
)
|
|
|
|
if not gainers_result['success']:
|
|
logger.error(f"❌ 52-week gainers error: {gainers_result.get('error')}")
|
|
raise HTTPException(
|
|
status_code=503,
|
|
detail=f"Failed to fetch 52-week gainers: {gainers_result.get('error', 'Unknown error')}"
|
|
)
|
|
|
|
# Extract data
|
|
most_active_stocks = most_active_result['data']['stocks']
|
|
gainers_stocks = gainers_result['data']['stocks']
|
|
|
|
# Track timing
|
|
most_active_time = most_active_result['data'].get('elapsed_time_seconds', 0)
|
|
gainers_time = gainers_result['data'].get('elapsed_time_seconds', 0)
|
|
|
|
# Normalize and categorize stocks
|
|
trending_stocks = []
|
|
seen_symbols = set()
|
|
overlap_count = 0
|
|
|
|
# Add most active stocks
|
|
for i, stock in enumerate(most_active_stocks[:most_active_limit]):
|
|
symbol = stock.get('symbol', '').upper()
|
|
if symbol:
|
|
trending_stock = {
|
|
'symbol': symbol,
|
|
'company_name': stock.get('company_name', 'N/A'),
|
|
'current_price': stock.get('current_price', stock.get('price_raw', 'N/A')),
|
|
'change_amount': stock.get('change_amount', stock.get('change_raw', 'N/A')),
|
|
'change_percent': stock.get('change_percent', stock.get('change_percent_raw', 'N/A')),
|
|
'volume': stock.get('volume', stock.get('volume_raw', 'N/A')),
|
|
'category': 'most_active',
|
|
'rank_in_category': i + 1,
|
|
'scraped_at': stock.get('scraped_at')
|
|
}
|
|
|
|
# Add average volume if available
|
|
if 'avg_volume' in stock or 'avg_volume_raw' in stock:
|
|
trending_stock['avg_volume'] = stock.get('avg_volume', stock.get('avg_volume_raw'))
|
|
|
|
trending_stocks.append(trending_stock)
|
|
seen_symbols.add(symbol)
|
|
|
|
# Add 52-week gainers
|
|
for i, stock in enumerate(gainers_stocks[:gainers_limit]):
|
|
symbol = stock.get('symbol', '').upper()
|
|
if symbol:
|
|
# Check for overlap
|
|
is_overlap = symbol in seen_symbols
|
|
if is_overlap:
|
|
overlap_count += 1
|
|
# Find and update existing stock to indicate it's in both categories
|
|
for existing_stock in trending_stocks:
|
|
if existing_stock['symbol'] == symbol:
|
|
existing_stock['category'] = 'both'
|
|
existing_stock['gainer_rank'] = i + 1
|
|
# Add 52w high if available
|
|
if 'high_52w' in stock:
|
|
existing_stock['high_52w'] = stock['high_52w']
|
|
break
|
|
else:
|
|
trending_stock = {
|
|
'symbol': symbol,
|
|
'company_name': stock.get('company_name', 'N/A'),
|
|
'current_price': stock.get('current_price', 'N/A'),
|
|
'change_amount': stock.get('change_amount', 'N/A'),
|
|
'change_percent': stock.get('change_percent', 'N/A'),
|
|
'volume': stock.get('volume', 'N/A'),
|
|
'category': '52_week_gainer',
|
|
'rank_in_category': i + 1,
|
|
'scraped_at': stock.get('scraped_at')
|
|
}
|
|
|
|
# Add 52w high if available
|
|
if 'high_52w' in stock:
|
|
trending_stock['high_52w'] = stock['high_52w']
|
|
|
|
trending_stocks.append(trending_stock)
|
|
seen_symbols.add(symbol)
|
|
|
|
# Limit final result to n stocks (prioritize most_active, then gainers by rank)
|
|
if len(trending_stocks) > n:
|
|
# Sort to prioritize most_active and low ranks
|
|
trending_stocks.sort(key=lambda x: (
|
|
x['category'] != 'most_active', # most_active first
|
|
x['category'] == '52_week_gainer', # then gainers
|
|
x['rank_in_category'] # then by rank within category
|
|
))
|
|
trending_stocks = trending_stocks[:n]
|
|
|
|
# Calculate final metrics
|
|
total_time = (datetime.now() - start_time).total_seconds()
|
|
unique_symbols = len(set(stock['symbol'] for stock in trending_stocks))
|
|
most_active_count = len([s for s in trending_stocks if s['category'] in ['most_active', 'both']])
|
|
gainers_count = len([s for s in trending_stocks if s['category'] in ['52_week_gainer', 'both']])
|
|
|
|
logger.info(f"✅ Successfully returned {len(trending_stocks)} trending stocks (target: {n}, unique: {unique_symbols}) in {total_time:.1f}s")
|
|
|
|
return {
|
|
"success": True,
|
|
"message": f"Retrieved {len(trending_stocks)} trending stocks ({most_active_count} most active + {gainers_count} gainers) in {total_time:.1f}s",
|
|
"data": {
|
|
"trending_stocks": trending_stocks,
|
|
"summary": {
|
|
"total_stocks": len(trending_stocks),
|
|
"most_active_count": most_active_count,
|
|
"gainers_count": gainers_count,
|
|
"unique_symbols": unique_symbols,
|
|
"overlap_count": overlap_count
|
|
},
|
|
"performance": {
|
|
"elapsed_time_seconds": round(total_time, 1),
|
|
"most_active_time": round(most_active_time, 1),
|
|
"gainers_time": round(gainers_time, 1),
|
|
"parallel_execution": True
|
|
},
|
|
"scraped_at": datetime.now().isoformat()
|
|
},
|
|
"metadata": {
|
|
"sources": [
|
|
"finance.yahoo.com/markets/stocks/most-active/",
|
|
"finance.yahoo.com/markets/stocks/52-week-gainers/"
|
|
],
|
|
"method": "parallel_scraping_with_intelligent_rate_limiting",
|
|
"categories": ["most_active", "52_week_gainer", "both"],
|
|
"rate_limit_bypass": "curl_cffi_chrome_impersonation",
|
|
"deduplication": "symbol_based_with_category_merge"
|
|
}
|
|
}
|
|
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"❌ Unexpected error in get_trending_stocks: {e}")
|
|
raise HTTPException(
|
|
status_code=500,
|
|
detail=f"Internal server error while fetching trending stocks: {str(e)}"
|
|
) |