You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1241 lines
50 KiB
Python
1241 lines
50 KiB
Python
"""
|
|
Service for fetching and processing price data from Yahoo Finance using yfinance-plus
|
|
"""
|
|
|
|
from datetime import datetime, timezone, timedelta, date
|
|
from typing import Dict, List, Optional, Tuple, Union
|
|
import logging
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
from sqlalchemy import select, and_, desc, or_
|
|
import asyncio
|
|
import sys
|
|
import os
|
|
|
|
# Add parent directory to path for imports
|
|
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
|
|
|
from app.core.database import AsyncSessionLocal
|
|
from app.models.financial import PriceData
|
|
from app.schemas.financial import DataSource, ErrorType
|
|
from app.utils.date_utils import parse_period, quarters_to_date_range, resolve_time_parameters
|
|
from app.core.config import settings
|
|
|
|
try:
|
|
import pandas as pd
|
|
except ImportError:
|
|
pd = None # type: ignore
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
async def _run_with_timeout(coro, timeout_seconds: float, description: str):
|
|
try:
|
|
return await asyncio.wait_for(coro, timeout=timeout_seconds)
|
|
except asyncio.TimeoutError:
|
|
raise TimeoutError(f"yfinance timed out after {timeout_seconds}s: {description}")
|
|
|
|
|
|
# Import yfinance-plus for price data only
|
|
try:
|
|
import yfinance_plus as yf
|
|
YFINANCE_AVAILABLE = True
|
|
logger.info("yfinance-plus imported successfully for price data")
|
|
except ImportError:
|
|
logger.error("yfinance-plus not available for price data")
|
|
YFINANCE_AVAILABLE = False
|
|
|
|
class PriceDataService:
|
|
def __init__(self):
|
|
self.yf_available = YFINANCE_AVAILABLE
|
|
if not self.yf_available:
|
|
logger.warning("Yahoo Finance (yfinance-plus) data will not be available")
|
|
|
|
async def get_or_update_price_data(
|
|
self,
|
|
ticker: str,
|
|
start_date: datetime,
|
|
end_date: datetime,
|
|
interval: str = "1d",
|
|
force_refresh: bool = False
|
|
) -> List[PriceData]:
|
|
"""
|
|
Get price data from database or fetch from Yahoo Finance if needed.
|
|
|
|
Session-per-phase: DB connections are held only during short DB operations,
|
|
never during yfinance calls (which can take 30s+).
|
|
|
|
Returns:
|
|
List of PriceData objects
|
|
"""
|
|
ticker = ticker.upper()
|
|
|
|
# Phase 1: check missing periods (short session)
|
|
async with AsyncSessionLocal() as db:
|
|
missing_periods = await self._check_missing_periods(
|
|
db, ticker, start_date, end_date, interval
|
|
)
|
|
|
|
if missing_periods or force_refresh:
|
|
if not self.yf_available:
|
|
raise ValueError("Yahoo Finance (yfinance-plus) data source not available")
|
|
|
|
# Phase 2: fetch from yfinance (no session held)
|
|
hist_data = await self._fetch_price_data(ticker, start_date, end_date, interval)
|
|
|
|
if hist_data is not None and not hist_data.empty:
|
|
# Phase 3: store in DB (short session)
|
|
async with AsyncSessionLocal() as db:
|
|
await self._store_price_data(db, ticker, hist_data, interval)
|
|
await db.commit()
|
|
|
|
# Phase 4: read from DB (short session)
|
|
async with AsyncSessionLocal() as db:
|
|
price_data = await self._get_price_data_from_db(
|
|
db, ticker, start_date, end_date, interval
|
|
)
|
|
|
|
return price_data
|
|
|
|
async def _check_missing_periods(
|
|
self,
|
|
db: AsyncSession,
|
|
ticker: str,
|
|
start_date: datetime,
|
|
end_date: datetime,
|
|
interval: str
|
|
) -> List[datetime]:
|
|
"""Check which periods are missing in the database.
|
|
|
|
Also treats today's data as always-missing so that end-of-day volumes
|
|
get re-fetched rather than returning a mid-session snapshot that was
|
|
cached earlier in the day.
|
|
"""
|
|
# If no date range provided, assume we need to fetch data
|
|
if start_date is None or end_date is None:
|
|
return [datetime.now()] # Return a dummy date to trigger fetch
|
|
|
|
# Always re-fetch if the range includes today: daily bars fetched mid-session
|
|
# are stored with partial (intraday) volume and must be refreshed after close.
|
|
today = datetime.now(timezone.utc).date()
|
|
if end_date.date() >= today:
|
|
return [datetime.now()]
|
|
|
|
# Count only rows with valid close (null/0 rows are treated as missing)
|
|
result = await db.execute(
|
|
select(PriceData.date)
|
|
.where(
|
|
and_(
|
|
PriceData.ticker == ticker,
|
|
PriceData.date >= start_date,
|
|
PriceData.date <= end_date,
|
|
PriceData.close.is_not(None),
|
|
PriceData.close > 0,
|
|
)
|
|
)
|
|
.order_by(PriceData.date)
|
|
)
|
|
|
|
existing_dates = {row[0].date() for row in result.fetchall()}
|
|
|
|
# Generate expected dates based on interval
|
|
expected_dates = self._generate_expected_dates(start_date, end_date, interval)
|
|
|
|
# Find missing dates
|
|
missing_dates = [date for date in expected_dates if date not in existing_dates]
|
|
|
|
# If more than 10% of dates are missing, consider it as needing refresh
|
|
if len(missing_dates) > len(expected_dates) * 0.1:
|
|
return missing_dates
|
|
|
|
return []
|
|
|
|
def _generate_expected_dates(
|
|
self,
|
|
start_date: datetime,
|
|
end_date: datetime,
|
|
interval: str
|
|
) -> List[datetime]:
|
|
"""Generate expected trading dates based on interval"""
|
|
expected_dates = []
|
|
current_date = start_date
|
|
|
|
# Simple date generation (doesn't account for market holidays)
|
|
if interval == "1d":
|
|
while current_date <= end_date:
|
|
# Skip weekends for daily data
|
|
if current_date.weekday() < 5: # Monday = 0, Friday = 4
|
|
expected_dates.append(current_date)
|
|
current_date += timedelta(days=1)
|
|
elif interval == "1w":
|
|
while current_date <= end_date:
|
|
expected_dates.append(current_date)
|
|
current_date += timedelta(weeks=1)
|
|
elif interval == "1m":
|
|
# Monthly data - first day of each month
|
|
while current_date <= end_date:
|
|
expected_dates.append(current_date)
|
|
# Move to next month
|
|
if current_date.month == 12:
|
|
current_date = current_date.replace(year=current_date.year + 1, month=1)
|
|
else:
|
|
current_date = current_date.replace(month=current_date.month + 1)
|
|
else:
|
|
# For other intervals, just return the date range
|
|
expected_dates = [start_date, end_date]
|
|
|
|
return expected_dates
|
|
|
|
async def _fetch_price_data(
|
|
self,
|
|
ticker: str,
|
|
start_date: datetime,
|
|
end_date: datetime,
|
|
interval: str
|
|
):
|
|
"""Fetch price data from Yahoo Finance (no DB operations).
|
|
|
|
Returns a DataFrame or None if no data was returned.
|
|
"""
|
|
logger.info(f"Fetching price data for {ticker} from {start_date} to {end_date}")
|
|
|
|
yf_ticker = yf.Ticker(ticker)
|
|
|
|
start_str = start_date.strftime('%Y-%m-%d')
|
|
# yfinance's `end` parameter is exclusive — add +1 day to include end_date.
|
|
end_inclusive = end_date + timedelta(days=1)
|
|
end_str = end_inclusive.strftime('%Y-%m-%d')
|
|
|
|
loop = asyncio.get_event_loop()
|
|
hist_data = await _run_with_timeout(
|
|
loop.run_in_executor(
|
|
None,
|
|
lambda: yf_ticker.history(
|
|
start=start_str,
|
|
end=end_str,
|
|
interval=interval,
|
|
auto_adjust=True,
|
|
prepost=False,
|
|
period=None
|
|
)
|
|
),
|
|
timeout_seconds=30,
|
|
description=f"history {ticker} {start_str}:{end_str}"
|
|
)
|
|
|
|
if hist_data is None or hist_data.empty:
|
|
logger.warning(f"No price data returned for {ticker}")
|
|
return None
|
|
|
|
logger.info(f"Fetched {len(hist_data)} price records for {ticker}")
|
|
return hist_data
|
|
|
|
async def get_quote(self, ticker: str, use_prepost: bool = True) -> Dict:
|
|
"""Get latest quote using yfinance-plus .info fields with fallback to fast_info + history."""
|
|
if not self.yf_available:
|
|
raise ValueError("Yahoo Finance (yfinance-plus) data source not available")
|
|
|
|
yf_ticker = yf.Ticker(ticker)
|
|
loop = asyncio.get_event_loop()
|
|
|
|
# Try .info first; fall back to fast_info + history on timeout/error
|
|
info = None
|
|
try:
|
|
info = await _run_with_timeout(
|
|
loop.run_in_executor(None, lambda: yf_ticker.info),
|
|
timeout_seconds=20,
|
|
description=f"info {ticker}"
|
|
)
|
|
except Exception as e:
|
|
logger.warning(f"get_quote .info failed for {ticker}: {e}, falling back to fast_info")
|
|
|
|
if info:
|
|
regular = info.get("regularMarketPrice")
|
|
post = info.get("postMarketPrice") if use_prepost else None
|
|
pre = info.get("preMarketPrice") if use_prepost else None
|
|
currency = info.get("currency")
|
|
exchange = info.get("exchange") or info.get("fullExchangeName")
|
|
market_state = info.get("marketState")
|
|
ts = info.get("regularMarketTime") or info.get("postMarketTime") or info.get("preMarketTime")
|
|
if isinstance(ts, (int, float)):
|
|
ts = datetime.fromtimestamp(ts, tz=timezone.utc)
|
|
elif isinstance(ts, datetime):
|
|
if ts.tzinfo is None:
|
|
ts = ts.replace(tzinfo=timezone.utc)
|
|
else:
|
|
ts = datetime.now(timezone.utc)
|
|
pre_val = float(pre) if pre is not None else None
|
|
post_val = float(post) if post is not None else None
|
|
regular_val = float(regular) if regular is not None else None
|
|
price_val = post_val or pre_val or regular_val
|
|
else:
|
|
# Fallback: fast_info for price/currency/exchange, history for timestamp
|
|
try:
|
|
fast = await _run_with_timeout(
|
|
loop.run_in_executor(None, lambda: yf_ticker.fast_info),
|
|
timeout_seconds=10,
|
|
description=f"fast_info {ticker}"
|
|
)
|
|
regular_val = getattr(fast, "last_price", None)
|
|
if regular_val is None and isinstance(fast, dict):
|
|
regular_val = fast.get("lastPrice") or fast.get("last_price")
|
|
currency = getattr(fast, "currency", None) or (fast.get("currency") if isinstance(fast, dict) else None)
|
|
exchange = getattr(fast, "exchange", None) or (fast.get("exchange") if isinstance(fast, dict) else None)
|
|
except Exception as e2:
|
|
logger.error(f"get_quote fast_info also failed for {ticker}: {e2}")
|
|
raise
|
|
|
|
# Try to get last close from recent history for timestamp
|
|
try:
|
|
df = await _run_with_timeout(
|
|
loop.run_in_executor(
|
|
None,
|
|
lambda: yf_ticker.history(period="2d", interval="1d", auto_adjust=True)
|
|
),
|
|
timeout_seconds=15,
|
|
description=f"history fallback {ticker}"
|
|
)
|
|
if not df.empty:
|
|
last_row = df.iloc[-1]
|
|
if regular_val is None:
|
|
regular_val = float(last_row.get("Close", 0)) or None
|
|
ts = df.index[-1].to_pydatetime()
|
|
if ts.tzinfo is None:
|
|
ts = ts.replace(tzinfo=timezone.utc)
|
|
else:
|
|
ts = datetime.now(timezone.utc)
|
|
except Exception:
|
|
ts = datetime.now(timezone.utc)
|
|
|
|
pre_val = None
|
|
post_val = None
|
|
price_val = regular_val
|
|
market_state = None
|
|
|
|
return {
|
|
"ticker": ticker.upper(),
|
|
"price": float(price_val) if price_val is not None else None,
|
|
"regular_price": float(regular_val) if regular_val is not None else None,
|
|
"pre_market_price": pre_val,
|
|
"post_market_price": post_val,
|
|
"currency": currency,
|
|
"exchange": exchange,
|
|
"market_state": market_state,
|
|
"timestamp": ts,
|
|
"source": DataSource.YAHOO_FINANCE,
|
|
"delayed": True,
|
|
}
|
|
|
|
async def get_intraday(self, ticker: str, interval: str = "1m", period: str = "1d") -> List[Dict]:
|
|
"""Get intraday candles using yfinance-plus history with period/interval."""
|
|
if not self.yf_available:
|
|
raise ValueError("Yahoo Finance (yfinance-plus) data source not available")
|
|
try:
|
|
yf_ticker = yf.Ticker(ticker)
|
|
loop = asyncio.get_event_loop()
|
|
df = await _run_with_timeout(
|
|
loop.run_in_executor(
|
|
None,
|
|
lambda: yf_ticker.history(period=period, interval=interval, auto_adjust=True, prepost=True)
|
|
),
|
|
timeout_seconds=30,
|
|
description=f"intraday {ticker} {period}/{interval}"
|
|
)
|
|
candles = []
|
|
if not df.empty:
|
|
for ts, row in df.iterrows():
|
|
dt = ts.to_pydatetime()
|
|
if dt.tzinfo is None:
|
|
dt = dt.replace(tzinfo=timezone.utc)
|
|
candles.append({
|
|
"timestamp": dt,
|
|
"open": float(row.get("Open", 0)) if not pd.isna(row.get("Open")) else None,
|
|
"high": float(row.get("High", 0)) if not pd.isna(row.get("High")) else None,
|
|
"low": float(row.get("Low", 0)) if not pd.isna(row.get("Low")) else None,
|
|
"close": float(row.get("Close", 0)) if not pd.isna(row.get("Close")) else 0.0,
|
|
"volume": float(row.get("Volume", 0)) if not pd.isna(row.get("Volume")) else None,
|
|
})
|
|
return candles
|
|
except Exception as e:
|
|
logger.error(f"Error fetching intraday for {ticker}: {str(e)}")
|
|
raise
|
|
|
|
async def get_multi_intraday(
|
|
self,
|
|
tickers: List[str],
|
|
interval: str = "5m",
|
|
start_date: Optional[date] = None,
|
|
end_date: Optional[date] = None,
|
|
chunk_size: int = 50,
|
|
) -> Dict[str, List[Dict]]:
|
|
"""
|
|
Fetch intraday bars for multiple tickers via yf.download().
|
|
|
|
Returns:
|
|
Dict mapping ticker → list of {timestamp, open, high, low, close, volume}
|
|
"""
|
|
if not self.yf_available:
|
|
raise ValueError("Yahoo Finance data source not available")
|
|
|
|
# end date for yfinance download must be exclusive (day after)
|
|
from datetime import timedelta
|
|
start_str = start_date.isoformat() if start_date else None
|
|
end_str = (end_date + timedelta(days=1)).isoformat() if end_date else None
|
|
|
|
result: Dict[str, List[Dict]] = {t.upper(): [] for t in tickers}
|
|
loop = asyncio.get_event_loop()
|
|
|
|
for i in range(0, len(tickers), chunk_size):
|
|
chunk = [t.upper() for t in tickers[i : i + chunk_size]]
|
|
_tickers_str = " ".join(chunk)
|
|
|
|
try:
|
|
bulk_data = await _run_with_timeout(
|
|
loop.run_in_executor(
|
|
None,
|
|
lambda ts=_tickers_str: yf.download(
|
|
tickers=ts,
|
|
start=start_str,
|
|
end=end_str,
|
|
interval=interval,
|
|
auto_adjust=True,
|
|
prepost=False,
|
|
group_by="ticker",
|
|
threads=True,
|
|
progress=False,
|
|
),
|
|
),
|
|
timeout_seconds=120,
|
|
description=f"multi_intraday chunk {i//chunk_size+1}",
|
|
)
|
|
except Exception as e:
|
|
logger.error(f"multi_intraday chunk error: {e}")
|
|
continue
|
|
|
|
if bulk_data is None or bulk_data.empty:
|
|
continue
|
|
|
|
def _parse_row(row):
|
|
def _f(v):
|
|
try:
|
|
return None if pd.isna(v) else float(v)
|
|
except Exception:
|
|
return None
|
|
|
|
return {
|
|
"open": _f(row.get("Open")),
|
|
"high": _f(row.get("High")),
|
|
"low": _f(row.get("Low")),
|
|
"close": _f(row.get("Close")) or 0.0,
|
|
"volume": _f(row.get("Volume")),
|
|
}
|
|
|
|
if len(chunk) == 1:
|
|
# Single-ticker: flat DataFrame
|
|
ticker = chunk[0]
|
|
for ts, row in bulk_data.iterrows():
|
|
dt = ts.to_pydatetime()
|
|
if dt.tzinfo is None:
|
|
dt = dt.replace(tzinfo=timezone.utc)
|
|
result[ticker].append({"timestamp": dt.isoformat(), **_parse_row(row)})
|
|
else:
|
|
# Multi-ticker: MultiIndex columns grouped by ticker
|
|
for ticker in chunk:
|
|
try:
|
|
lvl0 = bulk_data.columns.get_level_values(0)
|
|
if ticker not in lvl0:
|
|
continue
|
|
ticker_df = bulk_data[ticker]
|
|
for ts, row in ticker_df.iterrows():
|
|
dt = ts.to_pydatetime()
|
|
if dt.tzinfo is None:
|
|
dt = dt.replace(tzinfo=timezone.utc)
|
|
result[ticker].append({"timestamp": dt.isoformat(), **_parse_row(row)})
|
|
except Exception as e:
|
|
logger.error(f"multi_intraday parse error for {ticker}: {e}")
|
|
|
|
await asyncio.sleep(0.1) # rate-limit courtesy
|
|
|
|
return result
|
|
|
|
async def get_today_ohlc(self, ticker: str) -> Dict:
|
|
"""Get today's OHLC. If daily not yet finalized, aggregate from intraday 1m."""
|
|
if not self.yf_available:
|
|
raise ValueError("Yahoo Finance (yfinance-plus) data source not available")
|
|
try:
|
|
# First try daily with period=1d
|
|
yf_ticker = yf.Ticker(ticker)
|
|
loop = asyncio.get_event_loop()
|
|
daily = await _run_with_timeout(
|
|
loop.run_in_executor(
|
|
None, lambda: yf_ticker.history(period="1d", interval="1d", auto_adjust=True, prepost=False)
|
|
),
|
|
timeout_seconds=30,
|
|
description=f"today_ohlc {ticker}"
|
|
)
|
|
if daily is not None and not daily.empty:
|
|
ts, row = list(daily.iterrows())[-1]
|
|
d = ts.to_pydatetime().date()
|
|
return {
|
|
"ticker": ticker.upper(),
|
|
"date": d,
|
|
"open": float(row.get("Open", 0)) if not pd.isna(row.get("Open")) else None,
|
|
"high": float(row.get("High", 0)) if not pd.isna(row.get("High")) else None,
|
|
"low": float(row.get("Low", 0)) if not pd.isna(row.get("Low")) else None,
|
|
"close": float(row.get("Close", 0)) if not pd.isna(row.get("Close")) else 0.0,
|
|
"volume": float(row.get("Volume", 0)) if not pd.isna(row.get("Volume")) else None,
|
|
"source": DataSource.YAHOO_FINANCE,
|
|
"method": "daily",
|
|
}
|
|
# Fallback to intraday aggregation
|
|
intraday = await self.get_intraday(ticker, interval="1m", period="1d")
|
|
if not intraday:
|
|
raise ValueError("No intraday data available for today")
|
|
o = next((c["open"] for c in intraday if c.get("open") is not None), None)
|
|
h = max((c.get("high") or c.get("close") or 0.0) for c in intraday)
|
|
l = min((c.get("low") or c.get("close") or float("inf")) for c in intraday)
|
|
c = next((candle.get("close") for candle in reversed(intraday) if candle.get("close") is not None), 0.0)
|
|
v = sum((c.get("volume") or 0.0) for c in intraday)
|
|
today_date = intraday[0]["timestamp"].date()
|
|
return {
|
|
"ticker": ticker.upper(),
|
|
"date": today_date,
|
|
"open": o,
|
|
"high": h if h != 0.0 else None,
|
|
"low": l if l != float("inf") else None,
|
|
"close": c,
|
|
"volume": v or None,
|
|
"source": DataSource.YAHOO_FINANCE,
|
|
"method": "intraday_aggregate",
|
|
}
|
|
except Exception as e:
|
|
logger.error(f"Error fetching today OHLC for {ticker}: {str(e)}")
|
|
raise
|
|
|
|
async def _store_price_data(
|
|
self,
|
|
db: AsyncSession,
|
|
ticker: str,
|
|
hist_data,
|
|
interval: str
|
|
):
|
|
"""Store price data in database using batch upsert (INSERT ... ON CONFLICT DO NOTHING)."""
|
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
|
import uuid as _uuid
|
|
|
|
def _safe(val):
|
|
"""Return float or None, handling NaN/None safely."""
|
|
if val is None:
|
|
return None
|
|
try:
|
|
v = float(val)
|
|
return None if pd.isna(v) else v
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
now = datetime.now(timezone.utc)
|
|
today = now.date()
|
|
historical_rows = []
|
|
live_rows = []
|
|
for date_idx, row in hist_data.iterrows():
|
|
price_date = date_idx.to_pydatetime()
|
|
if price_date.tzinfo is None:
|
|
price_date = price_date.replace(tzinfo=timezone.utc)
|
|
# Normalize to UTC midnight so uq_price_data(ticker, date) deduplicates
|
|
# correctly regardless of whether data came from yf.Ticker().history()
|
|
# (returns Eastern midnight = UTC 04:00) or yf.download() (returns UTC 00:00).
|
|
price_date = price_date.replace(hour=0, minute=0, second=0, microsecond=0,
|
|
tzinfo=timezone.utc)
|
|
|
|
close_val = _safe(row.get('Close'))
|
|
if close_val is None: # Skip rows with no valid close price
|
|
continue
|
|
|
|
target_rows = live_rows if price_date.date() >= today else historical_rows
|
|
target_rows.append({
|
|
'id': _uuid.uuid4(),
|
|
'ticker': ticker,
|
|
'date': price_date,
|
|
'open': _safe(row.get('Open')),
|
|
'high': _safe(row.get('High')),
|
|
'low': _safe(row.get('Low')),
|
|
'close': close_val,
|
|
'volume': _safe(row.get('Volume')),
|
|
'adjusted_close': close_val,
|
|
'data_source': DataSource.YAHOO_FINANCE.value,
|
|
'created_at': now,
|
|
'updated_at': now,
|
|
})
|
|
|
|
if not historical_rows and not live_rows:
|
|
return
|
|
|
|
# Historical rows: preserve valid data but overwrite null/zero-close garbage
|
|
# (yf.download MultiIndex parse failures leave close=0 rows that must self-heal).
|
|
if historical_rows:
|
|
stmt = pg_insert(PriceData).values(historical_rows)
|
|
stmt = stmt.on_conflict_do_update(
|
|
constraint='uq_price_data',
|
|
set_={
|
|
'open': stmt.excluded.open,
|
|
'high': stmt.excluded.high,
|
|
'low': stmt.excluded.low,
|
|
'close': stmt.excluded.close,
|
|
'volume': stmt.excluded.volume,
|
|
'adjusted_close': stmt.excluded.adjusted_close,
|
|
'updated_at': stmt.excluded.updated_at,
|
|
},
|
|
where=or_(PriceData.close.is_(None), PriceData.close == 0.0),
|
|
)
|
|
await db.execute(stmt)
|
|
|
|
if live_rows:
|
|
stmt = pg_insert(PriceData).values(live_rows)
|
|
stmt = stmt.on_conflict_do_update(
|
|
constraint='uq_price_data',
|
|
set_={
|
|
'open': stmt.excluded.open,
|
|
'high': stmt.excluded.high,
|
|
'low': stmt.excluded.low,
|
|
'close': stmt.excluded.close,
|
|
'volume': stmt.excluded.volume,
|
|
'adjusted_close': stmt.excluded.adjusted_close,
|
|
'data_source': stmt.excluded.data_source,
|
|
'updated_at': stmt.excluded.updated_at,
|
|
}
|
|
)
|
|
await db.execute(stmt)
|
|
|
|
async def _get_price_data_from_db(
|
|
self,
|
|
db: AsyncSession,
|
|
ticker: str,
|
|
start_date: datetime,
|
|
end_date: datetime,
|
|
interval: str
|
|
) -> List[PriceData]:
|
|
"""Get price data from database"""
|
|
# Build query conditions
|
|
conditions = [PriceData.ticker == ticker]
|
|
|
|
if start_date is not None:
|
|
conditions.append(PriceData.date >= start_date)
|
|
if end_date is not None:
|
|
conditions.append(PriceData.date <= end_date)
|
|
|
|
result = await db.execute(
|
|
select(PriceData)
|
|
.where(and_(*conditions))
|
|
.order_by(PriceData.date)
|
|
)
|
|
|
|
return result.scalars().all()
|
|
|
|
async def get_latest_price(
|
|
self,
|
|
db: AsyncSession,
|
|
ticker: str
|
|
) -> Optional[PriceData]:
|
|
"""Get the latest price for a ticker"""
|
|
result = await db.execute(
|
|
select(PriceData)
|
|
.where(PriceData.ticker == ticker.upper())
|
|
.order_by(desc(PriceData.date))
|
|
.limit(1)
|
|
)
|
|
|
|
return result.scalar_one_or_none()
|
|
|
|
async def get_ticker_info(self, ticker: str) -> Dict:
|
|
"""Get ticker information from Yahoo Finance using yfinance-plus"""
|
|
if not self.yf_available:
|
|
raise ValueError("Yahoo Finance (yfinance-plus) data source not available")
|
|
|
|
try:
|
|
yf_ticker = yf.Ticker(ticker)
|
|
|
|
# Run in executor to avoid blocking
|
|
loop = asyncio.get_event_loop()
|
|
info = await _run_with_timeout(
|
|
loop.run_in_executor(None, lambda: yf_ticker.info),
|
|
timeout_seconds=20,
|
|
description=f"ticker_info {ticker}"
|
|
)
|
|
|
|
return info
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error fetching ticker info for {ticker}: {str(e)}")
|
|
raise
|
|
|
|
async def get_multiple_tickers_data_optimized(
|
|
self,
|
|
tickers: List[str],
|
|
start_date: datetime,
|
|
end_date: datetime,
|
|
interval: str = "1d",
|
|
force_refresh: bool = False
|
|
) -> Tuple[List, int, int]:
|
|
"""
|
|
Optimized bulk processing for multiple tickers with chunking for 100+ tickers:
|
|
1. Smart chunking to handle 100+ tickers efficiently
|
|
2. Parallel processing using asyncio with concurrency limits
|
|
3. Bulk yfinance queries using yfinance-plus bulk features
|
|
4. Optimized database operations with batch processing
|
|
5. Progress tracking for large requests
|
|
|
|
Supports unlimited ticker count with intelligent chunking:
|
|
- Small batches (≤50): Process in single chunk
|
|
- Medium batches (51-200): Process in 2-4 chunks
|
|
- Large batches (200+): Process in optimal chunks with progress tracking
|
|
|
|
Returns:
|
|
Tuple of (results, successful_count, failed_count) for API response
|
|
"""
|
|
from app.schemas.financial import BulkPriceDataItem, PriceDataResponse, PriceDataPoint
|
|
|
|
results = []
|
|
successful_count = 0
|
|
failed_count = 0
|
|
|
|
# Normalize tickers and validate
|
|
tickers = [t.upper().strip() for t in tickers if t.strip()]
|
|
total_tickers = len(tickers)
|
|
|
|
logger.info(f"Starting bulk processing for {total_tickers} tickers")
|
|
|
|
# Determine optimal chunking strategy based on ticker count
|
|
if total_tickers <= 50:
|
|
chunk_size = total_tickers # Single chunk for small requests
|
|
max_concurrent = 1
|
|
elif total_tickers <= 200:
|
|
chunk_size = 50 # Moderate chunks for medium requests
|
|
max_concurrent = 4
|
|
else:
|
|
chunk_size = 75 # Larger chunks for big requests
|
|
max_concurrent = 6
|
|
|
|
try:
|
|
# Process tickers in chunks to avoid overwhelming APIs and memory
|
|
all_results = []
|
|
all_successful = 0
|
|
all_failed = 0
|
|
|
|
for chunk_start in range(0, total_tickers, chunk_size):
|
|
chunk_end = min(chunk_start + chunk_size, total_tickers)
|
|
chunk_tickers = tickers[chunk_start:chunk_end]
|
|
chunk_num = (chunk_start // chunk_size) + 1
|
|
total_chunks = (total_tickers + chunk_size - 1) // chunk_size
|
|
|
|
logger.info(f"Processing chunk {chunk_num}/{total_chunks}: {len(chunk_tickers)} tickers")
|
|
|
|
# Phase 1: batch check missing periods (short session)
|
|
missing_tickers = []
|
|
if force_refresh:
|
|
missing_tickers = chunk_tickers.copy()
|
|
else:
|
|
async with AsyncSessionLocal() as db:
|
|
missing_tickers = await self._batch_check_missing_periods(
|
|
db, chunk_tickers, start_date, end_date, interval
|
|
)
|
|
|
|
# Phase 2: bulk yfinance fetch (no session held)
|
|
if missing_tickers and self.yf_available:
|
|
logger.info(f"Bulk fetching price data for {len(missing_tickers)} tickers in chunk {chunk_num}")
|
|
chunk_data_list = await self._bulk_fetch_price_data(
|
|
missing_tickers, start_date, end_date, interval
|
|
)
|
|
# Phase 3: store each sub-chunk with its own short-lived session
|
|
for sub_chunk_tickers, bulk_data in chunk_data_list:
|
|
async with AsyncSessionLocal() as db:
|
|
await self._process_bulk_data(db, sub_chunk_tickers, bulk_data, interval)
|
|
await db.commit()
|
|
|
|
# Phase 4: batch retrieve all data from DB (short session)
|
|
async with AsyncSessionLocal() as db:
|
|
ticker_data_map = await self._batch_get_price_data_from_db(
|
|
db, chunk_tickers, start_date, end_date, interval
|
|
)
|
|
|
|
# Step 4: Process results for this chunk
|
|
chunk_results = []
|
|
chunk_successful = 0
|
|
chunk_failed = 0
|
|
|
|
for ticker in chunk_tickers:
|
|
try:
|
|
price_data = ticker_data_map.get(ticker, [])
|
|
|
|
# Convert to response models
|
|
price_points = [
|
|
PriceDataPoint.model_construct(
|
|
date=pd.date.date() if isinstance(pd.date, datetime) else pd.date,
|
|
open=pd.open, high=pd.high, low=pd.low, close=pd.close,
|
|
volume=pd.volume, adjusted_close=pd.adjusted_close,
|
|
data_source=pd.data_source.value if hasattr(pd.data_source, 'value') else pd.data_source,
|
|
) for pd in price_data
|
|
]
|
|
|
|
# Calculate actual date range from returned data
|
|
actual_start_date = start_date
|
|
actual_end_date = end_date
|
|
|
|
if price_points:
|
|
# Get actual start and end dates from the data
|
|
actual_start_date = min(point.date for point in price_points)
|
|
actual_end_date = max(point.date for point in price_points)
|
|
|
|
response = PriceDataResponse(
|
|
ticker=ticker,
|
|
interval=interval,
|
|
data=price_points,
|
|
metadata={
|
|
"request_id": str(ticker),
|
|
"data_points": len(price_points),
|
|
"interval": interval,
|
|
"date_range": {
|
|
"start": actual_start_date.isoformat(),
|
|
"end": actual_end_date.isoformat()
|
|
},
|
|
"last_updated": datetime.now(timezone.utc).isoformat()
|
|
}
|
|
)
|
|
|
|
chunk_results.append(BulkPriceDataItem(
|
|
ticker=ticker,
|
|
success=True,
|
|
data=response,
|
|
error=None
|
|
))
|
|
chunk_successful += 1
|
|
|
|
except Exception as e:
|
|
# Handle individual ticker failure
|
|
error_message = str(e)
|
|
if "No price data found" in error_message or "No data returned" in error_message:
|
|
error_message = f"No price data found for ticker {ticker}"
|
|
elif "Invalid ticker" in error_message:
|
|
error_message = f"Invalid or unknown ticker: {ticker}"
|
|
elif "Yahoo Finance data source not available" in error_message:
|
|
error_message = "Yahoo Finance data source not available"
|
|
|
|
chunk_results.append(BulkPriceDataItem(
|
|
ticker=ticker,
|
|
success=False,
|
|
data=None,
|
|
error=error_message
|
|
))
|
|
chunk_failed += 1
|
|
|
|
# Aggregate chunk results
|
|
all_results.extend(chunk_results)
|
|
all_successful += chunk_successful
|
|
all_failed += chunk_failed
|
|
|
|
logger.info(f"Chunk {chunk_num} completed: {chunk_successful} successful, {chunk_failed} failed")
|
|
|
|
# Small delay between chunks to avoid overwhelming APIs
|
|
if chunk_num < total_chunks:
|
|
await asyncio.sleep(0.2)
|
|
|
|
logger.info(f"Bulk processing completed: {all_successful} successful, {all_failed} failed out of {total_tickers} total")
|
|
return all_results, all_successful, all_failed
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error in optimized bulk processing: {str(e)}")
|
|
# Fallback to individual processing
|
|
return await self._fallback_individual_processing(
|
|
tickers, start_date, end_date, interval, force_refresh
|
|
)
|
|
|
|
async def _batch_check_missing_periods(
|
|
self,
|
|
db: AsyncSession,
|
|
tickers: List[str],
|
|
start_date: datetime,
|
|
end_date: datetime,
|
|
interval: str
|
|
) -> List[str]:
|
|
"""Batch check which tickers have missing periods.
|
|
|
|
If the range includes today, all tickers are treated as missing so that
|
|
mid-session cached rows (partial volume) are always refreshed.
|
|
"""
|
|
# Always re-fetch when range includes today (same logic as _check_missing_periods)
|
|
today = datetime.now(timezone.utc).date()
|
|
if end_date.date() >= today:
|
|
logger.debug(f"_batch_check_missing_periods: end_date includes today — forcing re-fetch for all {len(tickers)} tickers")
|
|
return list(tickers)
|
|
|
|
# Single query to check all tickers at once — count only valid rows
|
|
from sqlalchemy import func, case
|
|
|
|
result = await db.execute(
|
|
select(
|
|
PriceData.ticker,
|
|
func.count(PriceData.date).label('count'),
|
|
func.min(PriceData.date).label('min_date'),
|
|
func.max(PriceData.date).label('max_date')
|
|
)
|
|
.where(
|
|
and_(
|
|
PriceData.ticker.in_(tickers),
|
|
PriceData.date >= start_date,
|
|
PriceData.date <= end_date,
|
|
PriceData.close.is_not(None),
|
|
PriceData.close > 0,
|
|
)
|
|
)
|
|
.group_by(PriceData.ticker)
|
|
)
|
|
|
|
existing_tickers = {}
|
|
for row in result.fetchall():
|
|
ticker, count, min_date, max_date = row
|
|
existing_tickers[ticker] = {
|
|
'count': count,
|
|
'min_date': min_date,
|
|
'max_date': max_date
|
|
}
|
|
|
|
# Determine expected count based on interval
|
|
expected_days = (end_date - start_date).days
|
|
if interval == "1d":
|
|
expected_count = expected_days * 0.7 # Rough estimate for trading days
|
|
elif interval == "1w":
|
|
expected_count = expected_days / 7
|
|
else:
|
|
expected_count = 1
|
|
|
|
missing_tickers = []
|
|
for ticker in tickers:
|
|
ticker_data = existing_tickers.get(ticker)
|
|
if not ticker_data or ticker_data['count'] < expected_count * 0.8:
|
|
missing_tickers.append(ticker)
|
|
|
|
logger.info(f"Found {len(missing_tickers)} tickers needing data refresh out of {len(tickers)}")
|
|
return missing_tickers
|
|
|
|
async def _bulk_fetch_price_data(
|
|
self,
|
|
tickers: List[str],
|
|
start_date: datetime,
|
|
end_date: datetime,
|
|
interval: str
|
|
) -> List[Tuple[List[str], object]]:
|
|
"""Fetch bulk price data from yfinance (no DB operations).
|
|
|
|
Returns a list of (chunk_tickers, bulk_data) tuples for the caller to
|
|
store with short-lived sessions.
|
|
"""
|
|
results = []
|
|
start_str = start_date.strftime('%Y-%m-%d')
|
|
# yfinance end is exclusive — add +1 day to include end_date (same as _fetch_price_data)
|
|
end_str = (end_date + timedelta(days=1)).strftime('%Y-%m-%d')
|
|
|
|
loop = asyncio.get_event_loop()
|
|
|
|
total_tickers = len(tickers)
|
|
if total_tickers <= 10:
|
|
chunk_size = total_tickers
|
|
elif total_tickers <= 50:
|
|
chunk_size = 15
|
|
else:
|
|
chunk_size = 20
|
|
|
|
logger.info(f"Starting bulk fetch for {total_tickers} tickers")
|
|
|
|
for i in range(0, total_tickers, chunk_size):
|
|
chunk_tickers = tickers[i:i + chunk_size]
|
|
_chunk_str = ' '.join(chunk_tickers)
|
|
|
|
logger.info(f"Fetching chunk {i//chunk_size + 1}: {len(chunk_tickers)} tickers")
|
|
|
|
try:
|
|
bulk_data = await _run_with_timeout(
|
|
loop.run_in_executor(
|
|
None,
|
|
# lambda default arg binds _chunk_str at definition time (closure bug fix)
|
|
lambda cs=_chunk_str: yf.download(
|
|
tickers=cs,
|
|
start=start_str,
|
|
end=end_str,
|
|
interval=interval,
|
|
auto_adjust=True,
|
|
prepost=False,
|
|
group_by='ticker',
|
|
threads=True,
|
|
)
|
|
),
|
|
timeout_seconds=60,
|
|
description=f"bulk_download {len(chunk_tickers)} tickers"
|
|
)
|
|
results.append((chunk_tickers, bulk_data))
|
|
except Exception as e:
|
|
logger.error(f"Error fetching chunk {i//chunk_size + 1}: {e}")
|
|
|
|
await asyncio.sleep(0.1)
|
|
|
|
logger.info(f"Completed bulk fetch for {total_tickers} tickers ({len(results)} chunks succeeded)")
|
|
return results
|
|
|
|
async def _process_bulk_data(
|
|
self,
|
|
db: AsyncSession,
|
|
tickers: List[str],
|
|
bulk_data,
|
|
interval: str
|
|
):
|
|
"""Process bulk data returned from yfinance and store in database"""
|
|
if bulk_data.empty:
|
|
logger.warning("No bulk data returned from yfinance")
|
|
return
|
|
|
|
# Handle different data structures from yfinance bulk download
|
|
if len(tickers) == 1:
|
|
# yf.download with group_by='ticker' returns MultiIndex columns even for a
|
|
# single ticker: [('SGOV', 'Open'), ('SGOV', 'Close'), ...]. Extract the
|
|
# ticker slice so _store_ticker_data receives a flat DataFrame.
|
|
ticker = tickers[0]
|
|
if hasattr(bulk_data.columns, 'levels') and ticker in bulk_data.columns.get_level_values(0):
|
|
ticker_data = bulk_data[ticker]
|
|
else:
|
|
ticker_data = bulk_data
|
|
await self._store_ticker_data(db, ticker, ticker_data, interval)
|
|
else:
|
|
# Multiple tickers - data is grouped by ticker
|
|
for ticker in tickers:
|
|
try:
|
|
if ticker in bulk_data.columns.get_level_values(0):
|
|
ticker_data = bulk_data[ticker]
|
|
if not ticker_data.empty:
|
|
await self._store_ticker_data(db, ticker, ticker_data, interval)
|
|
except Exception as e:
|
|
logger.error(f"Error processing data for {ticker}: {str(e)}")
|
|
continue
|
|
|
|
async def _store_ticker_data(
|
|
self,
|
|
db: AsyncSession,
|
|
ticker: str,
|
|
ticker_data,
|
|
interval: str
|
|
):
|
|
"""Store individual ticker data using batch upsert (INSERT ... ON CONFLICT DO NOTHING)."""
|
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
|
import uuid as _uuid
|
|
|
|
def _safe(val):
|
|
if val is None:
|
|
return None
|
|
try:
|
|
v = float(val)
|
|
return None if pd.isna(v) else v
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
now = datetime.now(timezone.utc)
|
|
today = now.date()
|
|
historical_rows = []
|
|
live_rows = []
|
|
for date_idx, row in ticker_data.iterrows():
|
|
price_date = date_idx.to_pydatetime()
|
|
if price_date.tzinfo is None:
|
|
price_date = price_date.replace(tzinfo=timezone.utc)
|
|
# Normalize to UTC midnight (same logic as _store_price_data)
|
|
price_date = price_date.replace(hour=0, minute=0, second=0, microsecond=0,
|
|
tzinfo=timezone.utc)
|
|
|
|
close_val = _safe(row.get('Close'))
|
|
if close_val is None: # Skip rows with no valid close price
|
|
continue
|
|
|
|
target_rows = live_rows if price_date.date() >= today else historical_rows
|
|
target_rows.append({
|
|
'id': _uuid.uuid4(),
|
|
'ticker': ticker,
|
|
'date': price_date,
|
|
'open': _safe(row.get('Open')),
|
|
'high': _safe(row.get('High')),
|
|
'low': _safe(row.get('Low')),
|
|
'close': close_val,
|
|
'volume': _safe(row.get('Volume')),
|
|
'adjusted_close': close_val,
|
|
'data_source': DataSource.YAHOO_FINANCE.value,
|
|
'created_at': now,
|
|
'updated_at': now,
|
|
})
|
|
|
|
if not historical_rows and not live_rows:
|
|
return
|
|
|
|
if historical_rows:
|
|
stmt = pg_insert(PriceData).values(historical_rows)
|
|
stmt = stmt.on_conflict_do_update(
|
|
constraint='uq_price_data',
|
|
set_={
|
|
'open': stmt.excluded.open,
|
|
'high': stmt.excluded.high,
|
|
'low': stmt.excluded.low,
|
|
'close': stmt.excluded.close,
|
|
'volume': stmt.excluded.volume,
|
|
'adjusted_close': stmt.excluded.adjusted_close,
|
|
'updated_at': stmt.excluded.updated_at,
|
|
},
|
|
where=or_(PriceData.close.is_(None), PriceData.close == 0.0),
|
|
)
|
|
await db.execute(stmt)
|
|
|
|
if live_rows:
|
|
stmt = pg_insert(PriceData).values(live_rows)
|
|
stmt = stmt.on_conflict_do_update(
|
|
constraint='uq_price_data',
|
|
set_={
|
|
'open': stmt.excluded.open,
|
|
'high': stmt.excluded.high,
|
|
'low': stmt.excluded.low,
|
|
'close': stmt.excluded.close,
|
|
'volume': stmt.excluded.volume,
|
|
'adjusted_close': stmt.excluded.adjusted_close,
|
|
'data_source': stmt.excluded.data_source,
|
|
'updated_at': stmt.excluded.updated_at,
|
|
}
|
|
)
|
|
await db.execute(stmt)
|
|
logger.info(
|
|
"Stored price records for %s: historical=%d live=%d",
|
|
ticker,
|
|
len(historical_rows),
|
|
len(live_rows),
|
|
)
|
|
|
|
async def _get_existing_dates_for_ticker(
|
|
self,
|
|
db: AsyncSession,
|
|
ticker: str
|
|
) -> set:
|
|
"""Get existing dates for a ticker to avoid duplicates"""
|
|
result = await db.execute(
|
|
select(PriceData.date)
|
|
.where(PriceData.ticker == ticker)
|
|
)
|
|
return {row[0].date() for row in result.fetchall()}
|
|
|
|
async def _batch_get_price_data_from_db(
|
|
self,
|
|
db: AsyncSession,
|
|
tickers: List[str],
|
|
start_date: datetime,
|
|
end_date: datetime,
|
|
interval: str
|
|
) -> Dict[str, List[PriceData]]:
|
|
"""Batch retrieve price data for multiple tickers"""
|
|
# Single query to get data for all tickers
|
|
result = await db.execute(
|
|
select(PriceData)
|
|
.where(
|
|
and_(
|
|
PriceData.ticker.in_(tickers),
|
|
PriceData.date >= start_date,
|
|
PriceData.date <= end_date
|
|
)
|
|
)
|
|
.order_by(PriceData.ticker, PriceData.date)
|
|
)
|
|
|
|
# Group results by ticker
|
|
ticker_data_map = {}
|
|
for ticker in tickers:
|
|
ticker_data_map[ticker] = []
|
|
|
|
for record in result.scalars().all():
|
|
if record.ticker in ticker_data_map:
|
|
ticker_data_map[record.ticker].append(record)
|
|
|
|
return ticker_data_map
|
|
|
|
async def _fallback_individual_processing(
|
|
self,
|
|
tickers: List[str],
|
|
start_date: datetime,
|
|
end_date: datetime,
|
|
interval: str,
|
|
force_refresh: bool
|
|
) -> Tuple[List, int, int]:
|
|
"""Fallback to individual processing if bulk processing fails"""
|
|
from app.schemas.financial import BulkPriceDataItem, PriceDataResponse, PriceDataPoint
|
|
|
|
logger.warning("Falling back to individual ticker processing")
|
|
|
|
results = []
|
|
successful_count = 0
|
|
failed_count = 0
|
|
|
|
for ticker in tickers:
|
|
try:
|
|
# Get price data
|
|
price_data = await self.get_or_update_price_data(
|
|
ticker, start_date, end_date, interval, force_refresh
|
|
)
|
|
|
|
# Convert to response models
|
|
price_points = [
|
|
PriceDataPoint.model_construct(
|
|
date=pd.date.date() if isinstance(pd.date, datetime) else pd.date,
|
|
open=pd.open, high=pd.high, low=pd.low, close=pd.close,
|
|
volume=pd.volume, adjusted_close=pd.adjusted_close,
|
|
data_source=pd.data_source.value if hasattr(pd.data_source, 'value') else pd.data_source,
|
|
) for pd in price_data
|
|
]
|
|
|
|
# Calculate actual date range from returned data
|
|
actual_start_date = start_date
|
|
actual_end_date = end_date
|
|
|
|
if price_points:
|
|
actual_start_date = min(point.date for point in price_points)
|
|
actual_end_date = max(point.date for point in price_points)
|
|
|
|
response = PriceDataResponse(
|
|
ticker=ticker,
|
|
interval=interval,
|
|
data=price_points,
|
|
metadata={
|
|
"request_id": str(ticker),
|
|
"data_points": len(price_points),
|
|
"interval": interval,
|
|
"date_range": {
|
|
"start": actual_start_date.isoformat(),
|
|
"end": actual_end_date.isoformat()
|
|
},
|
|
"last_updated": datetime.now(timezone.utc).isoformat()
|
|
}
|
|
)
|
|
|
|
results.append(BulkPriceDataItem(
|
|
ticker=ticker,
|
|
success=True,
|
|
data=response,
|
|
error=None
|
|
))
|
|
successful_count += 1
|
|
|
|
except Exception as e:
|
|
error_message = str(e)
|
|
results.append(BulkPriceDataItem(
|
|
ticker=ticker,
|
|
success=False,
|
|
data=None,
|
|
error=error_message
|
|
))
|
|
failed_count += 1
|
|
|
|
return results, successful_count, failed_count
|
|
|
|
|
|
# Import pandas for data processing
|
|
try:
|
|
import pandas as pd
|
|
except ImportError:
|
|
logger.error("pandas not available - price data service will not work")
|
|
pd = None
|