fix: 과거 가격 데이터 upsert → insert-only로 변경 (Yahoo adjusted price 덮어쓰기 방지)

과거 OHLC 행은 on_conflict_do_nothing, 오늘 행만 on_conflict_do_update.
Yahoo Finance의 배당·분할 소급 조정으로 인한 기존 PIT 데이터 훼손 방지.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
main
I Luk Kim 3 months ago
parent 79164ecb66
commit e2ac9fa4d4

@ -480,13 +480,16 @@ class PriceDataService:
return None return None
now = datetime.now(timezone.utc) now = datetime.now(timezone.utc)
rows = [] today = now.date()
historical_rows = []
live_rows = []
for date_idx, row in hist_data.iterrows(): for date_idx, row in hist_data.iterrows():
price_date = date_idx.to_pydatetime() price_date = date_idx.to_pydatetime()
if price_date.tzinfo is None: if price_date.tzinfo is None:
price_date = price_date.replace(tzinfo=timezone.utc) price_date = price_date.replace(tzinfo=timezone.utc)
rows.append({ target_rows = live_rows if price_date.date() >= today else historical_rows
target_rows.append({
'id': _uuid.uuid4(), 'id': _uuid.uuid4(),
'ticker': ticker, 'ticker': ticker,
'date': price_date, 'date': price_date,
@ -501,26 +504,34 @@ class PriceDataService:
'updated_at': now, 'updated_at': now,
}) })
if not rows: if not historical_rows and not live_rows:
return return
# Upsert: on conflict update OHLCV + updated_at so that stale intraday-cached rows # Yahoo-adjusted historical prices are not point-in-time stable: future
# get overwritten when force_refresh=True re-fetches final EOD data. # dividends/splits can rewrite old OHLC values. Preserve existing
stmt = pg_insert(PriceData).values(rows) # historical rows and only update today's row, where intraday partials
stmt = stmt.on_conflict_do_update( # legitimately need EOD replacement.
constraint='uq_price_data', if historical_rows:
set_={ stmt = pg_insert(PriceData).values(historical_rows)
'open': stmt.excluded.open, stmt = stmt.on_conflict_do_nothing(constraint='uq_price_data')
'high': stmt.excluded.high, await db.execute(stmt)
'low': stmt.excluded.low,
'close': stmt.excluded.close, if live_rows:
'volume': stmt.excluded.volume, stmt = pg_insert(PriceData).values(live_rows)
'adjusted_close': stmt.excluded.adjusted_close, stmt = stmt.on_conflict_do_update(
'data_source': stmt.excluded.data_source, constraint='uq_price_data',
'updated_at': stmt.excluded.updated_at, set_={
} 'open': stmt.excluded.open,
) 'high': stmt.excluded.high,
await db.execute(stmt) 'low': stmt.excluded.low,
'close': stmt.excluded.close,
'volume': stmt.excluded.volume,
'adjusted_close': stmt.excluded.adjusted_close,
'data_source': stmt.excluded.data_source,
'updated_at': stmt.excluded.updated_at,
}
)
await db.execute(stmt)
async def _get_price_data_from_db( async def _get_price_data_from_db(
self, self,
@ -942,13 +953,16 @@ class PriceDataService:
return None return None
now = datetime.now(timezone.utc) now = datetime.now(timezone.utc)
rows = [] today = now.date()
historical_rows = []
live_rows = []
for date_idx, row in ticker_data.iterrows(): for date_idx, row in ticker_data.iterrows():
price_date = date_idx.to_pydatetime() price_date = date_idx.to_pydatetime()
if price_date.tzinfo is None: if price_date.tzinfo is None:
price_date = price_date.replace(tzinfo=timezone.utc) price_date = price_date.replace(tzinfo=timezone.utc)
rows.append({ target_rows = live_rows if price_date.date() >= today else historical_rows
target_rows.append({
'id': _uuid.uuid4(), 'id': _uuid.uuid4(),
'ticker': ticker, 'ticker': ticker,
'date': price_date, 'date': price_date,
@ -963,25 +977,36 @@ class PriceDataService:
'updated_at': now, 'updated_at': now,
}) })
if not rows: if not historical_rows and not live_rows:
return return
stmt = pg_insert(PriceData).values(rows) if historical_rows:
stmt = stmt.on_conflict_do_update( stmt = pg_insert(PriceData).values(historical_rows)
constraint='uq_price_data', stmt = stmt.on_conflict_do_nothing(constraint='uq_price_data')
set_={ await db.execute(stmt)
'open': stmt.excluded.open,
'high': stmt.excluded.high, if live_rows:
'low': stmt.excluded.low, stmt = pg_insert(PriceData).values(live_rows)
'close': stmt.excluded.close, stmt = stmt.on_conflict_do_update(
'volume': stmt.excluded.volume, constraint='uq_price_data',
'adjusted_close': stmt.excluded.adjusted_close, set_={
'data_source': stmt.excluded.data_source, 'open': stmt.excluded.open,
'updated_at': stmt.excluded.updated_at, 'high': stmt.excluded.high,
} 'low': stmt.excluded.low,
'close': stmt.excluded.close,
'volume': stmt.excluded.volume,
'adjusted_close': stmt.excluded.adjusted_close,
'data_source': stmt.excluded.data_source,
'updated_at': stmt.excluded.updated_at,
}
)
await db.execute(stmt)
logger.info(
"Stored price records for %s: historical_insert_only=%d live_upsert=%d",
ticker,
len(historical_rows),
len(live_rows),
) )
await db.execute(stmt)
logger.info(f"Batch upserted {len(rows)} price records for {ticker}")
async def _get_existing_dates_for_ticker( async def _get_existing_dates_for_ticker(
self, self,
@ -1112,4 +1137,4 @@ try:
import pandas as pd import pandas as pd
except ImportError: except ImportError:
logger.error("pandas not available - price data service will not work") logger.error("pandas not available - price data service will not work")
pd = None pd = None

@ -1 +1 @@
Subproject commit 674b7e45c8c79f3617dee9cffdd3f1438e4852cf Subproject commit e7011cabd7cd7853c3bc0fbd41601e004cc99ff4
Loading…
Cancel
Save