@ -480,13 +480,16 @@ class PriceDataService:
return None
return None
now = datetime . now ( timezone . utc )
now = datetime . now ( timezone . utc )
rows = [ ]
today = now . date ( )
historical_rows = [ ]
live_rows = [ ]
for date_idx , row in hist_data . iterrows ( ) :
for date_idx , row in hist_data . iterrows ( ) :
price_date = date_idx . to_pydatetime ( )
price_date = date_idx . to_pydatetime ( )
if price_date . tzinfo is None :
if price_date . tzinfo is None :
price_date = price_date . replace ( tzinfo = timezone . utc )
price_date = price_date . replace ( tzinfo = timezone . utc )
rows . append ( {
target_rows = live_rows if price_date . date ( ) > = today else historical_rows
target_rows . append ( {
' id ' : _uuid . uuid4 ( ) ,
' id ' : _uuid . uuid4 ( ) ,
' ticker ' : ticker ,
' ticker ' : ticker ,
' date ' : price_date ,
' date ' : price_date ,
@ -501,26 +504,34 @@ class PriceDataService:
' updated_at ' : now ,
' updated_at ' : now ,
} )
} )
if not rows:
if not historical_rows and not live_ rows:
return
return
# Upsert: on conflict update OHLCV + updated_at so that stale intraday-cached rows
# Yahoo-adjusted historical prices are not point-in-time stable: future
# get overwritten when force_refresh=True re-fetches final EOD data.
# dividends/splits can rewrite old OHLC values. Preserve existing
stmt = pg_insert ( PriceData ) . values ( rows )
# historical rows and only update today's row, where intraday partials
stmt = stmt . on_conflict_do_update (
# legitimately need EOD replacement.
constraint = ' uq_price_data ' ,
if historical_rows :
set_ = {
stmt = pg_insert ( PriceData ) . values ( historical_rows )
' open ' : stmt . excluded . open ,
stmt = stmt . on_conflict_do_nothing ( constraint = ' uq_price_data ' )
' high ' : stmt . excluded . high ,
await db . execute ( stmt )
' low ' : stmt . excluded . low ,
' close ' : stmt . excluded . close ,
if live_rows :
' volume ' : stmt . excluded . volume ,
stmt = pg_insert ( PriceData ) . values ( live_rows )
' adjusted_close ' : stmt . excluded . adjusted_close ,
stmt = stmt . on_conflict_do_update (
' data_source ' : stmt . excluded . data_source ,
constraint = ' uq_price_data ' ,
' updated_at ' : stmt . excluded . updated_at ,
set_ = {
}
' open ' : stmt . excluded . open ,
)
' high ' : stmt . excluded . high ,
await db . execute ( stmt )
' low ' : stmt . excluded . low ,
' close ' : stmt . excluded . close ,
' volume ' : stmt . excluded . volume ,
' adjusted_close ' : stmt . excluded . adjusted_close ,
' data_source ' : stmt . excluded . data_source ,
' updated_at ' : stmt . excluded . updated_at ,
}
)
await db . execute ( stmt )
async def _get_price_data_from_db (
async def _get_price_data_from_db (
self ,
self ,
@ -942,13 +953,16 @@ class PriceDataService:
return None
return None
now = datetime . now ( timezone . utc )
now = datetime . now ( timezone . utc )
rows = [ ]
today = now . date ( )
historical_rows = [ ]
live_rows = [ ]
for date_idx , row in ticker_data . iterrows ( ) :
for date_idx , row in ticker_data . iterrows ( ) :
price_date = date_idx . to_pydatetime ( )
price_date = date_idx . to_pydatetime ( )
if price_date . tzinfo is None :
if price_date . tzinfo is None :
price_date = price_date . replace ( tzinfo = timezone . utc )
price_date = price_date . replace ( tzinfo = timezone . utc )
rows . append ( {
target_rows = live_rows if price_date . date ( ) > = today else historical_rows
target_rows . append ( {
' id ' : _uuid . uuid4 ( ) ,
' id ' : _uuid . uuid4 ( ) ,
' ticker ' : ticker ,
' ticker ' : ticker ,
' date ' : price_date ,
' date ' : price_date ,
@ -963,25 +977,36 @@ class PriceDataService:
' updated_at ' : now ,
' updated_at ' : now ,
} )
} )
if not rows:
if not historical_rows and not live_ rows:
return
return
stmt = pg_insert ( PriceData ) . values ( rows )
if historical_rows :
stmt = stmt . on_conflict_do_update (
stmt = pg_insert ( PriceData ) . values ( historical_rows )
constraint = ' uq_price_data ' ,
stmt = stmt . on_conflict_do_nothing ( constraint = ' uq_price_data ' )
set_ = {
await db . execute ( stmt )
' open ' : stmt . excluded . open ,
' high ' : stmt . excluded . high ,
if live_rows :
' low ' : stmt . excluded . low ,
stmt = pg_insert ( PriceData ) . values ( live_rows )
' close ' : stmt . excluded . close ,
stmt = stmt . on_conflict_do_update (
' volume ' : stmt . excluded . volume ,
constraint = ' uq_price_data ' ,
' adjusted_close ' : stmt . excluded . adjusted_close ,
set_ = {
' data_source ' : stmt . excluded . data_source ,
' open ' : stmt . excluded . open ,
' updated_at ' : stmt . excluded . updated_at ,
' high ' : stmt . excluded . high ,
}
' low ' : stmt . excluded . low ,
' close ' : stmt . excluded . close ,
' volume ' : stmt . excluded . volume ,
' adjusted_close ' : stmt . excluded . adjusted_close ,
' data_source ' : stmt . excluded . data_source ,
' updated_at ' : stmt . excluded . updated_at ,
}
)
await db . execute ( stmt )
logger . info (
" Stored price records for %s : historical_insert_only= %d live_upsert= %d " ,
ticker ,
len ( historical_rows ) ,
len ( live_rows ) ,
)
)
await db . execute ( stmt )
logger . info ( f " Batch upserted { len ( rows ) } price records for { ticker } " )
async def _get_existing_dates_for_ticker (
async def _get_existing_dates_for_ticker (
self ,
self ,
@ -1112,4 +1137,4 @@ try:
import pandas as pd
import pandas as pd
except ImportError :
except ImportError :
logger . error ( " pandas not available - price data service will not work " )
logger . error ( " pandas not available - price data service will not work " )
pd = None
pd = None