diff --git a/app/services/universe_service.py b/app/services/universe_service.py index dc3e979..881f8e4 100644 --- a/app/services/universe_service.py +++ b/app/services/universe_service.py @@ -88,7 +88,12 @@ class UniverseService: No sector/industry (those remain NULL in registry). No market_cap filter (filter happens at screening time). """ - _US_EXCHANGES = {"NYSE", "NASDAQ", "AMEX", "ARCA", "BATS", "NYSEArca", "OTC"} + # Case-insensitive match; normalize to canonical uppercase form + _US_EXCHANGE_MAP = { + "nyse": "NYSE", "nasdaq": "NASDAQ", "amex": "AMEX", + "nysemkt": "AMEX", "arca": "ARCA", "nysearca": "ARCA", + "bats": "BATS", "otc": "OTC", + } try: data = await self._http.fetch_json( "https://www.sec.gov/files/company_tickers_exchange.json" @@ -110,11 +115,12 @@ class UniverseService: for row in rows_raw: try: ticker = str(row[ticker_idx]).upper().strip() - exchange = str(row[exchange_idx]).strip() + exchange_raw = str(row[exchange_idx]).strip() + canonical_exchange = _US_EXCHANGE_MAP.get(exchange_raw.lower()) if not ticker or len(ticker) > 10: continue - # Keep only major US exchanges - if exchange not in _US_EXCHANGES: + # Keep only major US exchanges (case-insensitive) + if not canonical_exchange: continue # Skip preferred stocks / warrants / rights / units (contain - or end in W/R/U/Z) if "-" in ticker: @@ -125,7 +131,7 @@ class UniverseService: "symbol": ticker, "shortName": str(row[name_idx]) if row[name_idx] else None, "cik_override": str(row[cik_idx]).zfill(10), - "exchange": exchange, + "exchange": canonical_exchange, "sector": None, "industry": None, "quoteType": "EQUITY", @@ -320,6 +326,7 @@ class UniverseService: total_snapshots = 0 total_failed = 0 total_batches = (len(ticker_list) + _PRICE_BATCH - 1) // _PRICE_BATCH + _CACHE_CLEAR_EVERY = 20 # clear SEC in-memory cache every N batches to prevent OOM # ---- Process in batches — each batch uses its own DB session ---- for batch_start in range(0, len(ticker_list), _PRICE_BATCH): @@ -425,6 +432,15 @@ class UniverseService: f"Universe: batch {batch_num}/{total_batches} — " f"{len(batch)} tickers, {len(batch_rows)} snapshot rows" ) + # Periodically clear SEC in-memory JSON cache to prevent OOM. + # Disk cache is retained — subsequent lookups re-load from disk. + if batch_num % _CACHE_CLEAR_EVERY == 0: + import gc + self._http._json_cache.clear() + self._http._text_cache.clear() + gc.collect() + logger.info(f"Universe: cleared SEC JSON cache at batch {batch_num}") + # Yield to event loop between batches to keep API responsive await asyncio.sleep(0)