diff --git a/app/services/universe_service.py b/app/services/universe_service.py index f58753b..e07b049 100644 --- a/app/services/universe_service.py +++ b/app/services/universe_service.py @@ -52,13 +52,8 @@ class UniverseService: sortField="intradaymarketcap", sortAsc=False, ) - async def discover_tickers(self, db: AsyncSession, market_cap_min: float = 1e8) -> Dict: - """ - Discover US-listed stocks via yfinance screener and store in registry. - Filters out ETFs, mutual funds, and foreign-listed stocks. - - Returns: { tickers_found, tickers_registered } - """ + async def _discover_via_yfinance(self, market_cap_min: float) -> List[dict]: + """Try yfinance screener. Returns list of quote dicts or raises on 401/error.""" from yfinance import EquityQuery query = EquityQuery("and", [ @@ -84,7 +79,83 @@ class UniverseService: break await asyncio.sleep(0.3) - logger.info(f"Universe: screener returned {len(all_quotes)} quotes") + return all_quotes + + async def _discover_via_sec_edgar(self) -> List[dict]: + """ + Fallback: use SEC EDGAR company_tickers_exchange.json. + Returns list of dicts with ticker/name/cik/exchange. + No sector/industry (those remain NULL in registry). + No market_cap filter (filter happens at screening time). + """ + _US_EXCHANGES = {"NYSE", "NASDAQ", "AMEX", "ARCA", "BATS", "NYSEArca", "OTC"} + try: + data = await self._http.fetch_json( + "https://www.sec.gov/files/company_tickers_exchange.json" + ) + except Exception as e: + raise RuntimeError(f"SEC EDGAR company_tickers_exchange.json fetch failed: {e}") + + fields = data.get("fields", []) + rows_raw = data.get("data", []) + try: + cik_idx = fields.index("cik") + name_idx = fields.index("name") + ticker_idx = fields.index("ticker") + exchange_idx = fields.index("exchange") + except ValueError as e: + raise RuntimeError(f"Unexpected company_tickers_exchange.json schema: {e}") + + quotes = [] + for row in rows_raw: + try: + ticker = str(row[ticker_idx]).upper().strip() + exchange = str(row[exchange_idx]).strip() + if not ticker or len(ticker) > 10: + continue + # Keep only major US exchanges + if exchange not in _US_EXCHANGES: + continue + quotes.append({ + "symbol": ticker, + "shortName": str(row[name_idx]) if row[name_idx] else None, + "cik_override": str(row[cik_idx]).zfill(10), + "exchange": exchange, + "sector": None, + "industry": None, + "quoteType": "EQUITY", + }) + except (IndexError, TypeError): + continue + + logger.info(f"Universe: SEC EDGAR fallback returned {len(quotes)} tickers") + return quotes + + async def discover_tickers(self, db: AsyncSession, market_cap_min: float = 1e8) -> Dict: + """ + Discover US-listed stocks and store in registry. + + Primary: yfinance screener (includes market_cap filter + sector/industry) + Fallback: SEC EDGAR company_tickers_exchange.json (no market_cap filter, + no sector/industry — those remain NULL) + + Returns: { tickers_found, tickers_registered, source } + """ + source = "yfinance" + all_quotes: List[dict] = [] + + try: + all_quotes = await self._discover_via_yfinance(market_cap_min) + if not all_quotes: + raise RuntimeError("yfinance screener returned 0 results") + logger.info(f"Universe: yfinance screener returned {len(all_quotes)} quotes") + except Exception as e: + logger.warning( + f"Universe: yfinance screener failed ({e}), " + "falling back to SEC EDGAR company_tickers_exchange.json" + ) + source = "sec_edgar" + all_quotes = await self._discover_via_sec_edgar() # Fetch CIK map from SEC once (disk-cached after first call) cik_map = await self._fetch_cik_map() @@ -106,7 +177,7 @@ class UniverseService: rows.append({ "ticker": ticker, "name": q.get("shortName") or q.get("longName"), - "cik": cik_map.get(ticker), + "cik": q.get("cik_override") or cik_map.get(ticker), "sector": q.get("sector"), "industry": q.get("industry"), "exchange": _REVERSE_EXCHANGE.get(exchange_code, exchange_code) or None, @@ -114,7 +185,7 @@ class UniverseService: }) if not rows: - return {"tickers_found": len(all_quotes), "tickers_registered": 0} + return {"tickers_found": len(all_quotes), "tickers_registered": 0, "source": source} for i in range(0, len(rows), _CHUNK): chunk = rows[i:i + _CHUNK] @@ -134,8 +205,8 @@ class UniverseService: await db.execute(stmt) await db.commit() - logger.info(f"Universe: upserted {len(rows)} tickers into registry") - return {"tickers_found": len(all_quotes), "tickers_registered": len(rows)} + logger.info(f"Universe: upserted {len(rows)} tickers into registry (source={source})") + return {"tickers_found": len(all_quotes), "tickers_registered": len(rows), "source": source} async def _fetch_cik_map(self) -> Dict[str, str]: """Fetch SEC company_tickers.json once → {TICKER: padded_cik}."""