@ -14,6 +14,7 @@ import os
# Add parent directory to path for imports # Add parent directory to path for imports
sys . path . append ( os . path . dirname ( os . path . dirname ( os . path . dirname ( os . path . abspath ( __file__ ) ) ) ) ) sys . path . append ( os . path . dirname ( os . path . dirname ( os . path . dirname ( os . path . abspath ( __file__ ) ) ) ) )
from app . core . database import AsyncSessionLocal
from app . models . financial import PriceData from app . models . financial import PriceData
from app . schemas . financial import DataSource , ErrorType from app . schemas . financial import DataSource , ErrorType
from app . utils . date_utils import parse_period , quarters_to_date_range , resolve_time_parameters from app . utils . date_utils import parse_period , quarters_to_date_range , resolve_time_parameters
@ -51,7 +52,6 @@ class PriceDataService:
async def get_or_update_price_data (
async def get_or_update_price_data (
self ,
self ,
db : AsyncSession ,
ticker : str ,
ticker : str ,
start_date : datetime ,
start_date : datetime ,
end_date : datetime ,
end_date : datetime ,
@ -59,40 +59,41 @@ class PriceDataService:
force_refresh : bool = False
force_refresh : bool = False
) - > List [ PriceData ] :
) - > List [ PriceData ] :
"""
"""
Get price data from database or fetch from Yahoo Finance if needed
Get price data from database or fetch from Yahoo Finance if needed .
Args :
Session - per - phase : DB connections are held only during short DB operations ,
db : Database session
never during yfinance calls ( which can take 30 s + ) .
ticker : Stock ticker symbol
start_date : Start date for data retrieval
end_date : End date for data retrieval
interval : Data interval ( 1 d , 1 w , 1 m , 1 h , etc . )
force_refresh : Force refresh data from Yahoo Finance
Returns :
Returns :
List of PriceData objects
List of PriceData objects
"""
"""
ticker = ticker . upper ( )
ticker = ticker . upper ( )
# Check if we need to fetch new data
# Phase 1: check missing periods (short session)
missing_periods = await self . _check_missing_periods (
async with AsyncSessionLocal ( ) as db :
db , ticker , start_date , end_date , interval
missing_periods = await self . _check_missing_periods (
)
db , ticker , start_date , end_date , interval
)
if missing_periods or force_refresh :
if missing_periods or force_refresh :
if not self . yf_available :
if not self . yf_available :
raise ValueError ( " Yahoo Finance (yfinance-plus) data source not available " )
raise ValueError ( " Yahoo Finance (yfinance-plus) data source not available " )
# Fetch data from Yahoo Finance using yfinance-plus
# Phase 2: fetch from yfinance (no session held)
await self . _fetch_and_store_price_data (
hist_data = await self . _fetch_price_data ( ticker , start_date , end_date , interval )
if hist_data is not None and not hist_data . empty :
# Phase 3: store in DB (short session)
async with AsyncSessionLocal ( ) as db :
await self . _store_price_data ( db , ticker , hist_data , interval )
await db . commit ( )
# Phase 4: read from DB (short session)
async with AsyncSessionLocal ( ) as db :
price_data = await self . _get_price_data_from_db (
db , ticker , start_date , end_date , interval
db , ticker , start_date , end_date , interval
)
)
# Retrieve data from database
price_data = await self . _get_price_data_from_db (
db , ticker , start_date , end_date , interval
)
return price_data
return price_data
async def _check_missing_periods (
async def _check_missing_periods (
@ -182,63 +183,49 @@ class PriceDataService:
return expected_dates
return expected_dates
async def _fetch_ and_store_ price_data(
async def _fetch_ price_data(
self ,
self ,
db : AsyncSession ,
ticker : str ,
ticker : str ,
start_date : datetime ,
start_date : datetime ,
end_date : datetime ,
end_date : datetime ,
interval : str
interval : str
) :
) :
""" Fetch price data from Yahoo Finance using yfinance-plus and store in database """
""" Fetch price data from Yahoo Finance (no DB operations).
try :
logger . info ( f " Fetching price data for { ticker } from { start_date } to { end_date } " )
Returns a DataFrame or None if no data was returned .
"""
# Create yfinance-plus ticker object
logger . info ( f " Fetching price data for { ticker } from { start_date } to { end_date } " )
yf_ticker = yf . Ticker ( ticker )
yf_ticker = yf . Ticker ( ticker )
# Fetch historical data
# Convert dates to strings in YYYY-MM-DD format
start_str = start_date . strftime ( ' % Y- % m- %d ' )
start_str = start_date . strftime ( ' % Y- % m- %d ' )
# yfinance's `end` parameter is exclusive — add +1 day to include end_date.
# yfinance's `end` parameter is exclusive for daily data when using date strings.
end_inclusive = end_date + timedelta ( days = 1 )
# Add +1 day to include the intended end_date day in the results.
end_str = end_inclusive . strftime ( ' % Y- % m- %d ' )
from datetime import timedelta
end_inclusive = end_date + timedelta ( days = 1 )
loop = asyncio . get_event_loop ( )
end_str = end_inclusive . strftime ( ' % Y- % m- %d ' )
hist_data = await _run_with_timeout (
loop . run_in_executor (
# Run yfinance-plus in executor to avoid blocking
None ,
loop = asyncio . get_event_loop ( )
lambda : yf_ticker . history (
hist_data = await _run_with_timeout (
start = start_str ,
loop . run_in_executor (
end = end_str ,
None ,
interval = interval ,
lambda : yf_ticker . history (
auto_adjust = True ,
start = start_str ,
prepost = False ,
end = end_str ,
period = None
interval = interval ,
)
auto_adjust = True ,
) ,
prepost = False ,
timeout_seconds = 30 ,
period = None # Explicitly set period to None when using start/end dates
description = f " history { ticker } { start_str } : { end_str } "
)
)
) ,
timeout_seconds = 30 ,
if hist_data is None or hist_data . empty :
description = f " history { ticker } { start_str } : { end_str } "
logger . warning ( f " No price data returned for { ticker } " )
)
return None
if hist_data . empty :
logger . info ( f " Fetched { len ( hist_data ) } price records for { ticker } " )
logger . warning ( f " No price data returned for { ticker } " )
return hist_data
return
# Store data in database
await self . _store_price_data ( db , ticker , hist_data , interval )
await db . commit ( )
logger . info ( f " Successfully stored { len ( hist_data ) } price records for { ticker } " )
except Exception as e :
logger . error ( f " Error fetching price data for { ticker } : { str ( e ) } " )
await db . rollback ( )
raise
async def get_quote ( self , ticker : str , use_prepost : bool = True ) - > Dict :
async def get_quote ( self , ticker : str , use_prepost : bool = True ) - > Dict :
""" Get latest quote using yfinance-plus .info fields with fallback to fast history last row. """
""" Get latest quote using yfinance-plus .info fields with fallback to fast history last row. """
@ -597,33 +584,8 @@ class PriceDataService:
logger . error ( f " Error fetching ticker info for { ticker } : { str ( e ) } " )
logger . error ( f " Error fetching ticker info for { ticker } : { str ( e ) } " )
raise
raise
async def get_multiple_tickers_data (
self ,
db : AsyncSession ,
tickers : List [ str ] ,
start_date : datetime ,
end_date : datetime ,
interval : str = " 1d " ,
force_refresh : bool = False
) - > Dict [ str , List [ PriceData ] ] :
""" Get price data for multiple tickers (legacy method) """
results = { }
for ticker in tickers :
try :
data = await self . get_or_update_price_data (
db , ticker , start_date , end_date , interval , force_refresh
)
results [ ticker ] = data
except Exception as e :
logger . error ( f " Error fetching data for { ticker } : { str ( e ) } " )
results [ ticker ] = [ ]
return results
async def get_multiple_tickers_data_optimized (
async def get_multiple_tickers_data_optimized (
self ,
self ,
db : AsyncSession ,
tickers : List [ str ] ,
tickers : List [ str ] ,
start_date : datetime ,
start_date : datetime ,
end_date : datetime ,
end_date : datetime ,
@ -683,26 +645,33 @@ class PriceDataService:
logger . info ( f " Processing chunk { chunk_num } / { total_chunks } : { len ( chunk_tickers ) } tickers " )
logger . info ( f " Processing chunk { chunk_num } / { total_chunks } : { len ( chunk_tickers ) } tickers " )
# Step 1: Batch check missing periods for chunk
# Phase 1: batch check missing periods (short session)
missing_tickers = [ ]
missing_tickers = [ ]
if force_refresh :
if force_refresh :
missing_tickers = chunk_tickers . copy ( )
missing_tickers = chunk_tickers . copy ( )
else :
else :
missing_tickers = await self . _batch_check_missing_periods (
async with AsyncSessionLocal ( ) as db :
db , chunk_tickers , start_date , end_date , interval
missing_tickers = await self . _batch_check_missing_periods (
)
db , chunk_tickers , start_date , end_date , interval
)
# Step 2: If we have missing data, use bulk yfinance fetch
# Phase 2: bulk yfinance fetch (no session held)
if missing_tickers and self . yf_available :
if missing_tickers and self . yf_available :
logger . info ( f " Bulk fetching price data for { len ( missing_tickers ) } tickers in chunk { chunk_num } " )
logger . info ( f " Bulk fetching price data for { len ( missing_tickers ) } tickers in chunk { chunk_num } " )
await self . _bulk_fetch_and_store_price_data (
chunk_data_list = await self . _bulk_fetch_price_data (
db , missing_tickers , start_date , end_date , interval
missing_tickers , start_date , end_date , interval
)
# Phase 3: store each sub-chunk with its own short-lived session
for sub_chunk_tickers , bulk_data in chunk_data_list :
async with AsyncSessionLocal ( ) as db :
await self . _process_bulk_data ( db , sub_chunk_tickers , bulk_data , interval )
await db . commit ( )
# Phase 4: batch retrieve all data from DB (short session)
async with AsyncSessionLocal ( ) as db :
ticker_data_map = await self . _batch_get_price_data_from_db (
db , chunk_tickers , start_date , end_date , interval
)
)
# Step 3: Batch retrieve all data from database for this chunk
ticker_data_map = await self . _batch_get_price_data_from_db (
db , chunk_tickers , start_date , end_date , interval
)
# Step 4: Process results for this chunk
# Step 4: Process results for this chunk
chunk_results = [ ]
chunk_results = [ ]
@ -792,7 +761,7 @@ class PriceDataService:
logger . error ( f " Error in optimized bulk processing: { str ( e ) } " )
logger . error ( f " Error in optimized bulk processing: { str ( e ) } " )
# Fallback to individual processing
# Fallback to individual processing
return await self . _fallback_individual_processing (
return await self . _fallback_individual_processing (
db, tickers, start_date , end_date , interval , force_refresh
, start_date , end_date , interval , force_refresh
)
)
async def _batch_check_missing_periods (
async def _batch_check_missing_periods (
@ -861,72 +830,67 @@ class PriceDataService:
logger . info ( f " Found { len ( missing_tickers ) } tickers needing data refresh out of { len ( tickers ) } " )
logger . info ( f " Found { len ( missing_tickers ) } tickers needing data refresh out of { len ( tickers ) } " )
return missing_tickers
return missing_tickers
async def _bulk_fetch_ and_store_ price_data(
async def _bulk_fetch_ price_data(
self ,
self ,
db : AsyncSession ,
tickers : List [ str ] ,
tickers : List [ str ] ,
start_date : datetime ,
start_date : datetime ,
end_date : datetime ,
end_date : datetime ,
interval : str
interval : str
) :
) - > List [ Tuple [ List [ str ] , object ] ] :
""" Optimized bulk fetch using yfinance-plus bulk features """
""" Fetch bulk price data from yfinance (no DB operations).
try :
logger . info ( f " Starting bulk fetch for { len ( tickers ) } tickers " )
Returns a list of ( chunk_tickers , bulk_data ) tuples for the caller to
store with short - lived sessions .
# Convert dates to strings
"""
start_str = start_date . strftime ( ' % Y- % m- %d ' )
results = [ ]
end_str = end_date . strftime ( ' % Y- % m- %d ' )
start_str = start_date . strftime ( ' % Y- % m- %d ' )
end_str = end_date . strftime ( ' % Y- % m- %d ' )
# Use yfinance-plus bulk download feature
loop = asyncio . get_event_loop ( )
loop = asyncio . get_event_loop ( )
# Use adaptive chunk size for yfinance API calls based on ticker count
total_tickers = len ( tickers )
# Smaller chunks for yfinance API calls to avoid overwhelming the service
if total_tickers < = 10 :
total_tickers = len ( tickers )
chunk_size = total_tickers
if total_tickers < = 10 :
elif total_tickers < = 50 :
chunk_size = total_tickers # Single chunk for very small batches
chunk_size = 15
elif total_tickers < = 50 :
else :
chunk_size = 15 # Small chunks for moderate batches
chunk_size = 20
else :
chunk_size = 20 # Standard chunks for large batches
logger . info ( f " Starting bulk fetch for { total_tickers } tickers " )
for i in range ( 0 , len ( tickers ) , chunk_size ) :
chunk_tickers = tickers [ i : i + chunk_size ]
for i in range ( 0 , total_tickers , chunk_size ) :
chunk_tickers = tickers [ i : i + chunk_size ]
logger . info ( f " Processing chunk { i / / chunk_size + 1 } : { len ( chunk_tickers ) } tickers " )
_chunk_str = ' ' . join ( chunk_tickers )
# Use yfinance-plus bulk download
logger . info ( f " Fetching chunk { i / / chunk_size + 1 } : { len ( chunk_tickers ) } tickers " )
_chunk_str = ' ' . join ( chunk_tickers )
try :
bulk_data = await _run_with_timeout (
bulk_data = await _run_with_timeout (
loop . run_in_executor (
loop . run_in_executor (
None ,
None ,
lambda : yf . download (
# lambda default arg binds _chunk_str at definition time (closure bug fix)
tickers = _chunk_str ,
lambda cs = _chunk_str : yf . download (
tickers = cs ,
start = start_str ,
start = start_str ,
end = end_str ,
end = end_str ,
interval = interval ,
interval = interval ,
auto_adjust = True ,
auto_adjust = True ,
prepost = False ,
prepost = False ,
group_by = ' ticker ' ,
group_by = ' ticker ' ,
threads = True # Enable multi-threading
threads = True ,
)
)
) ,
) ,
timeout_seconds = 60 ,
timeout_seconds = 60 ,
description = f " bulk_download { len ( chunk_tickers ) } tickers "
description = f " bulk_download { len ( chunk_tickers ) } tickers "
)
)
results . append ( ( chunk_tickers , bulk_data ) )
# Process and store data for each ticker in the chunk
except Exception as e :
await self . _process_bulk_data ( db , chunk_tickers , bulk_data , interval )
logger . error ( f " Error fetching chunk { i / / chunk_size + 1 } : { e } " )
# Small delay to be nice to the API
await asyncio . sleep ( 0.1 )
await asyncio . sleep ( 0.1 )
logger . info ( f " Completed bulk fetch for { total_tickers } tickers ( { len ( results ) } chunks succeeded) " )
await db . commit ( )
return results
logger . info ( f " Successfully completed bulk fetch for { len ( tickers ) } tickers " )
except Exception as e :
logger . error ( f " Error in bulk fetch: { str ( e ) } " )
await db . rollback ( )
raise
async def _process_bulk_data (
async def _process_bulk_data (
self ,
self ,
@ -1065,7 +1029,6 @@ class PriceDataService:
async def _fallback_individual_processing (
async def _fallback_individual_processing (
self ,
self ,
db : AsyncSession ,
tickers : List [ str ] ,
tickers : List [ str ] ,
start_date : datetime ,
start_date : datetime ,
end_date : datetime ,
end_date : datetime ,
@ -1074,18 +1037,18 @@ class PriceDataService:
) - > Tuple [ List , int , int ] :
) - > Tuple [ List , int , int ] :
""" Fallback to individual processing if bulk processing fails """
""" Fallback to individual processing if bulk processing fails """
from app . schemas . financial import BulkPriceDataItem , PriceDataResponse , PriceDataPoint
from app . schemas . financial import BulkPriceDataItem , PriceDataResponse , PriceDataPoint
logger . warning ( " Falling back to individual ticker processing " )
logger . warning ( " Falling back to individual ticker processing " )
results = [ ]
results = [ ]
successful_count = 0
successful_count = 0
failed_count = 0
failed_count = 0
for ticker in tickers :
for ticker in tickers :
try :
try :
# Get price data
# Get price data
price_data = await self . get_or_update_price_data (
price_data = await self . get_or_update_price_data (
db, ticker, start_date , end_date , interval , force_refresh
, start_date , end_date , interval , force_refresh
)
)
# Convert to response models
# Convert to response models
I Luk Kim
4 months ago