fix: universe 데이터 품질 개선 — sanity check + preferred stock 필터링

- SEC EDGAR fallback에서 "-" 포함 티커 제외 (preferred stock, BAC-PL 등)
- W/R/Z suffix 티커 제외 (warrant, right, special)
- build_snapshots: close > $1M 또는 shares < 100,000 또는 market_cap > $5T 제외
- screen_historical: market_cap > $5T 쿼리 레벨 필터 추가

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
main
I Luk Kim 5 months ago
parent 5ab1663977
commit b54ac6d5b7

@ -116,6 +116,11 @@ class UniverseService:
# Keep only major US exchanges # Keep only major US exchanges
if exchange not in _US_EXCHANGES: if exchange not in _US_EXCHANGES:
continue continue
# Skip preferred stocks / warrants / rights / units (contain - or end in W/R/U/Z)
if "-" in ticker:
continue
if len(ticker) > 1 and ticker[-1] in ("W", "R", "Z") and ticker[:-1].isalpha():
continue
quotes.append({ quotes.append({
"symbol": ticker, "symbol": ticker,
"shortName": str(row[name_idx]) if row[name_idx] else None, "shortName": str(row[name_idx]) if row[name_idx] else None,
@ -343,9 +348,14 @@ class UniverseService:
shares = latest_shares shares = latest_shares
if shares is None or close is None: if shares is None or close is None:
continue continue
# Sanity checks: skip absurd values (data quality)
if close > 1_000_000 or close <= 0: # max BRK-A ~$600K
continue
if shares < 100_000: # too few shares for a real public co
continue
market_cap = shares * close market_cap = shares * close
if math.isnan(market_cap) or market_cap <= 0: if math.isnan(market_cap) or market_cap <= 0 or market_cap > 5e12:
continue continue
# Normalize to first of month # Normalize to first of month
@ -537,7 +547,12 @@ class UniverseService:
snapshot_dt = datetime(target.year, target.month, 1, tzinfo=timezone.utc) snapshot_dt = datetime(target.year, target.month, 1, tzinfo=timezone.utc)
conditions = [UniverseSnapshot.snapshot_date == snapshot_dt] # Always exclude clearly bad data (sanity cap: $5T max, historical record is ~$3.7T)
_MAX_MARKET_CAP = 5e12
conditions = [
UniverseSnapshot.snapshot_date == snapshot_dt,
UniverseSnapshot.market_cap <= _MAX_MARKET_CAP,
]
if market_cap_min is not None: if market_cap_min is not None:
conditions.append(UniverseSnapshot.market_cap >= market_cap_min) conditions.append(UniverseSnapshot.market_cap >= market_cap_min)
if market_cap_max is not None: if market_cap_max is not None:

Loading…
Cancel
Save