@ -88,7 +88,12 @@ class UniverseService:
No sector / industry ( those remain NULL in registry ) .
No sector / industry ( those remain NULL in registry ) .
No market_cap filter ( filter happens at screening time ) .
No market_cap filter ( filter happens at screening time ) .
"""
"""
_US_EXCHANGES = { " NYSE " , " NASDAQ " , " AMEX " , " ARCA " , " BATS " , " NYSEArca " , " OTC " }
# Case-insensitive match; normalize to canonical uppercase form
_US_EXCHANGE_MAP = {
" nyse " : " NYSE " , " nasdaq " : " NASDAQ " , " amex " : " AMEX " ,
" nysemkt " : " AMEX " , " arca " : " ARCA " , " nysearca " : " ARCA " ,
" bats " : " BATS " , " otc " : " OTC " ,
}
try :
try :
data = await self . _http . fetch_json (
data = await self . _http . fetch_json (
" https://www.sec.gov/files/company_tickers_exchange.json "
" https://www.sec.gov/files/company_tickers_exchange.json "
@ -110,11 +115,12 @@ class UniverseService:
for row in rows_raw :
for row in rows_raw :
try :
try :
ticker = str ( row [ ticker_idx ] ) . upper ( ) . strip ( )
ticker = str ( row [ ticker_idx ] ) . upper ( ) . strip ( )
exchange = str ( row [ exchange_idx ] ) . strip ( )
exchange_raw = str ( row [ exchange_idx ] ) . strip ( )
canonical_exchange = _US_EXCHANGE_MAP . get ( exchange_raw . lower ( ) )
if not ticker or len ( ticker ) > 10 :
if not ticker or len ( ticker ) > 10 :
continue
continue
# Keep only major US exchanges
# Keep only major US exchanges (case-insensitive)
if exchange not in _US_EXCHANGES :
if not canonical_ exchange:
continue
continue
# Skip preferred stocks / warrants / rights / units (contain - or end in W/R/U/Z)
# Skip preferred stocks / warrants / rights / units (contain - or end in W/R/U/Z)
if " - " in ticker :
if " - " in ticker :
@ -125,7 +131,7 @@ class UniverseService:
" symbol " : ticker ,
" symbol " : ticker ,
" shortName " : str ( row [ name_idx ] ) if row [ name_idx ] else None ,
" shortName " : str ( row [ name_idx ] ) if row [ name_idx ] else None ,
" cik_override " : str ( row [ cik_idx ] ) . zfill ( 10 ) ,
" cik_override " : str ( row [ cik_idx ] ) . zfill ( 10 ) ,
" exchange " : exchange,
" exchange " : canonical_ exchange,
" sector " : None ,
" sector " : None ,
" industry " : None ,
" industry " : None ,
" quoteType " : " EQUITY " ,
" quoteType " : " EQUITY " ,
@ -320,6 +326,7 @@ class UniverseService:
total_snapshots = 0
total_snapshots = 0
total_failed = 0
total_failed = 0
total_batches = ( len ( ticker_list ) + _PRICE_BATCH - 1 ) / / _PRICE_BATCH
total_batches = ( len ( ticker_list ) + _PRICE_BATCH - 1 ) / / _PRICE_BATCH
_CACHE_CLEAR_EVERY = 20 # clear SEC in-memory cache every N batches to prevent OOM
# ---- Process in batches — each batch uses its own DB session ----
# ---- Process in batches — each batch uses its own DB session ----
for batch_start in range ( 0 , len ( ticker_list ) , _PRICE_BATCH ) :
for batch_start in range ( 0 , len ( ticker_list ) , _PRICE_BATCH ) :
@ -425,6 +432,15 @@ class UniverseService:
f " Universe: batch { batch_num } / { total_batches } — "
f " Universe: batch { batch_num } / { total_batches } — "
f " { len ( batch ) } tickers, { len ( batch_rows ) } snapshot rows "
f " { len ( batch ) } tickers, { len ( batch_rows ) } snapshot rows "
)
)
# Periodically clear SEC in-memory JSON cache to prevent OOM.
# Disk cache is retained — subsequent lookups re-load from disk.
if batch_num % _CACHE_CLEAR_EVERY == 0 :
import gc
self . _http . _json_cache . clear ( )
self . _http . _text_cache . clear ( )
gc . collect ( )
logger . info ( f " Universe: cleared SEC JSON cache at batch { batch_num } " )
# Yield to event loop between batches to keep API responsive
# Yield to event loop between batches to keep API responsive
await asyncio . sleep ( 0 )
await asyncio . sleep ( 0 )