fix(stocks): Wikipedia 403 우회를 위해 User-Agent 헤더 추가

pd.read_html() 대신 urllib로 HTML 직접 fetch 후 파싱.
User-Agent 미설정 시 Wikipedia가 403 반환하는 문제 수정.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
main
I Luk Kim 5 months ago
parent cbc0f93123
commit 969706de01

@ -5,7 +5,9 @@ S&P 500 / Nasdaq 100 구성 종목을 Wikipedia에서 조회
import asyncio
import logging
from io import StringIO
from typing import Optional
from urllib.request import Request, urlopen
import pandas as pd
@ -41,7 +43,10 @@ class IndexConstituentsService:
industry_col = config["industry_col"]
logger.info(f"Fetching Wikipedia tables from {url}")
tables = pd.read_html(url, flavor="lxml")
req = Request(url, headers={"User-Agent": "Mozilla/5.0 (compatible; StockOracle/1.0)"})
with urlopen(req, timeout=25) as resp:
html = resp.read().decode("utf-8")
tables = pd.read_html(StringIO(html), flavor="lxml")
# Try configured table index first
df = None

Loading…
Cancel
Save