From 5ab1663977ff2fb05acfbd15f17e872aeebc18c9 Mon Sep 17 00:00:00 2001 From: I Luk Kim Date: Sun, 29 Mar 2026 16:27:03 -0700 Subject: [PATCH] =?UTF-8?q?feat:=20universe=20discover=EC=97=90=20SEC=20ED?= =?UTF-8?q?GAR=20fallback=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit yf.screen() 401 에러 시 SEC EDGAR company_tickers_exchange.json으로 자동 전환. - 5,848개 NYSE/NASDAQ/AMEX/ARCA 종목 등록 가능 - response에 source 필드 추가 (yfinance / sec_edgar) - market_cap_min 필터는 yfinance 사용 시에만 적용 (SEC Edgar fallback 시 미적용, 스크리닝 시점에 필터링) Co-Authored-By: Claude Sonnet 4.6 --- app/services/universe_service.py | 95 ++++++++++++++++++++++++++++---- 1 file changed, 83 insertions(+), 12 deletions(-) diff --git a/app/services/universe_service.py b/app/services/universe_service.py index f58753b..e07b049 100644 --- a/app/services/universe_service.py +++ b/app/services/universe_service.py @@ -52,13 +52,8 @@ class UniverseService: sortField="intradaymarketcap", sortAsc=False, ) - async def discover_tickers(self, db: AsyncSession, market_cap_min: float = 1e8) -> Dict: - """ - Discover US-listed stocks via yfinance screener and store in registry. - Filters out ETFs, mutual funds, and foreign-listed stocks. - - Returns: { tickers_found, tickers_registered } - """ + async def _discover_via_yfinance(self, market_cap_min: float) -> List[dict]: + """Try yfinance screener. Returns list of quote dicts or raises on 401/error.""" from yfinance import EquityQuery query = EquityQuery("and", [ @@ -84,7 +79,83 @@ class UniverseService: break await asyncio.sleep(0.3) - logger.info(f"Universe: screener returned {len(all_quotes)} quotes") + return all_quotes + + async def _discover_via_sec_edgar(self) -> List[dict]: + """ + Fallback: use SEC EDGAR company_tickers_exchange.json. + Returns list of dicts with ticker/name/cik/exchange. + No sector/industry (those remain NULL in registry). + No market_cap filter (filter happens at screening time). + """ + _US_EXCHANGES = {"NYSE", "NASDAQ", "AMEX", "ARCA", "BATS", "NYSEArca", "OTC"} + try: + data = await self._http.fetch_json( + "https://www.sec.gov/files/company_tickers_exchange.json" + ) + except Exception as e: + raise RuntimeError(f"SEC EDGAR company_tickers_exchange.json fetch failed: {e}") + + fields = data.get("fields", []) + rows_raw = data.get("data", []) + try: + cik_idx = fields.index("cik") + name_idx = fields.index("name") + ticker_idx = fields.index("ticker") + exchange_idx = fields.index("exchange") + except ValueError as e: + raise RuntimeError(f"Unexpected company_tickers_exchange.json schema: {e}") + + quotes = [] + for row in rows_raw: + try: + ticker = str(row[ticker_idx]).upper().strip() + exchange = str(row[exchange_idx]).strip() + if not ticker or len(ticker) > 10: + continue + # Keep only major US exchanges + if exchange not in _US_EXCHANGES: + continue + quotes.append({ + "symbol": ticker, + "shortName": str(row[name_idx]) if row[name_idx] else None, + "cik_override": str(row[cik_idx]).zfill(10), + "exchange": exchange, + "sector": None, + "industry": None, + "quoteType": "EQUITY", + }) + except (IndexError, TypeError): + continue + + logger.info(f"Universe: SEC EDGAR fallback returned {len(quotes)} tickers") + return quotes + + async def discover_tickers(self, db: AsyncSession, market_cap_min: float = 1e8) -> Dict: + """ + Discover US-listed stocks and store in registry. + + Primary: yfinance screener (includes market_cap filter + sector/industry) + Fallback: SEC EDGAR company_tickers_exchange.json (no market_cap filter, + no sector/industry — those remain NULL) + + Returns: { tickers_found, tickers_registered, source } + """ + source = "yfinance" + all_quotes: List[dict] = [] + + try: + all_quotes = await self._discover_via_yfinance(market_cap_min) + if not all_quotes: + raise RuntimeError("yfinance screener returned 0 results") + logger.info(f"Universe: yfinance screener returned {len(all_quotes)} quotes") + except Exception as e: + logger.warning( + f"Universe: yfinance screener failed ({e}), " + "falling back to SEC EDGAR company_tickers_exchange.json" + ) + source = "sec_edgar" + all_quotes = await self._discover_via_sec_edgar() # Fetch CIK map from SEC once (disk-cached after first call) cik_map = await self._fetch_cik_map() @@ -106,7 +177,7 @@ class UniverseService: rows.append({ "ticker": ticker, "name": q.get("shortName") or q.get("longName"), - "cik": cik_map.get(ticker), + "cik": q.get("cik_override") or cik_map.get(ticker), "sector": q.get("sector"), "industry": q.get("industry"), "exchange": _REVERSE_EXCHANGE.get(exchange_code, exchange_code) or None, @@ -114,7 +185,7 @@ class UniverseService: }) if not rows: - return {"tickers_found": len(all_quotes), "tickers_registered": 0} + return {"tickers_found": len(all_quotes), "tickers_registered": 0, "source": source} for i in range(0, len(rows), _CHUNK): chunk = rows[i:i + _CHUNK] @@ -134,8 +205,8 @@ class UniverseService: await db.execute(stmt) await db.commit() - logger.info(f"Universe: upserted {len(rows)} tickers into registry") - return {"tickers_found": len(all_quotes), "tickers_registered": len(rows)} + logger.info(f"Universe: upserted {len(rows)} tickers into registry (source={source})") + return {"tickers_found": len(all_quotes), "tickers_registered": len(rows), "source": source} async def _fetch_cik_map(self) -> Dict[str, str]: """Fetch SEC company_tickers.json once → {TICKER: padded_cik}."""