fix: universe service - NASDAQ case bug + OOM prevention

Two fixes to universe snapshot build:

1. SEC EDGAR exchange case mismatch: company_tickers_exchange.json uses
   "Nasdaq" (mixed case) but filter expected "NASDAQ". All NASDAQ-listed
   stocks (AAPL, MSFT, GOOGL, etc.) were silently excluded from registry.
   Fixed with case-insensitive _US_EXCHANGE_MAP lookup + canonical normalization.

2. OOM during large builds: SEC EDGAR companyfacts JSONs accumulate in
   _json_cache without eviction, causing OOM after ~1500-1800 tickers.
   Fixed by clearing _json_cache + _text_cache + gc.collect() every 20
   batches. Memory remains stable throughout full 9,376-ticker build.

Result: 529,328 snapshot rows, 5,570 tickers (NASDAQ:2515, NYSE:2084, OTC:971)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
main
I Luk Kim 5 months ago
parent 24736954bb
commit c4565159b1

@ -88,7 +88,12 @@ class UniverseService:
No sector/industry (those remain NULL in registry).
No market_cap filter (filter happens at screening time).
"""
_US_EXCHANGES = {"NYSE", "NASDAQ", "AMEX", "ARCA", "BATS", "NYSEArca", "OTC"}
# Case-insensitive match; normalize to canonical uppercase form
_US_EXCHANGE_MAP = {
"nyse": "NYSE", "nasdaq": "NASDAQ", "amex": "AMEX",
"nysemkt": "AMEX", "arca": "ARCA", "nysearca": "ARCA",
"bats": "BATS", "otc": "OTC",
}
try:
data = await self._http.fetch_json(
"https://www.sec.gov/files/company_tickers_exchange.json"
@ -110,11 +115,12 @@ class UniverseService:
for row in rows_raw:
try:
ticker = str(row[ticker_idx]).upper().strip()
exchange = str(row[exchange_idx]).strip()
exchange_raw = str(row[exchange_idx]).strip()
canonical_exchange = _US_EXCHANGE_MAP.get(exchange_raw.lower())
if not ticker or len(ticker) > 10:
continue
# Keep only major US exchanges
if exchange not in _US_EXCHANGES:
# Keep only major US exchanges (case-insensitive)
if not canonical_exchange:
continue
# Skip preferred stocks / warrants / rights / units (contain - or end in W/R/U/Z)
if "-" in ticker:
@ -125,7 +131,7 @@ class UniverseService:
"symbol": ticker,
"shortName": str(row[name_idx]) if row[name_idx] else None,
"cik_override": str(row[cik_idx]).zfill(10),
"exchange": exchange,
"exchange": canonical_exchange,
"sector": None,
"industry": None,
"quoteType": "EQUITY",
@ -320,6 +326,7 @@ class UniverseService:
total_snapshots = 0
total_failed = 0
total_batches = (len(ticker_list) + _PRICE_BATCH - 1) // _PRICE_BATCH
_CACHE_CLEAR_EVERY = 20 # clear SEC in-memory cache every N batches to prevent OOM
# ---- Process in batches — each batch uses its own DB session ----
for batch_start in range(0, len(ticker_list), _PRICE_BATCH):
@ -425,6 +432,15 @@ class UniverseService:
f"Universe: batch {batch_num}/{total_batches}"
f"{len(batch)} tickers, {len(batch_rows)} snapshot rows"
)
# Periodically clear SEC in-memory JSON cache to prevent OOM.
# Disk cache is retained — subsequent lookups re-load from disk.
if batch_num % _CACHE_CLEAR_EVERY == 0:
import gc
self._http._json_cache.clear()
self._http._text_cache.clear()
gc.collect()
logger.info(f"Universe: cleared SEC JSON cache at batch {batch_num}")
# Yield to event loop between batches to keep API responsive
await asyncio.sleep(0)

Loading…
Cancel
Save