fix(stocks): Wikipedia 403 우회를 위해 User-Agent 헤더 추가

pd.read_html() 대신 urllib로 HTML 직접 fetch 후 파싱.
User-Agent 미설정 시 Wikipedia가 403 반환하는 문제 수정.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
main
I Luk Kim 5 months ago
parent cbc0f93123
commit 969706de01

@ -5,7 +5,9 @@ S&P 500 / Nasdaq 100 구성 종목을 Wikipedia에서 조회
import asyncio import asyncio
import logging import logging
from io import StringIO
from typing import Optional from typing import Optional
from urllib.request import Request, urlopen
import pandas as pd import pandas as pd
@ -41,7 +43,10 @@ class IndexConstituentsService:
industry_col = config["industry_col"] industry_col = config["industry_col"]
logger.info(f"Fetching Wikipedia tables from {url}") logger.info(f"Fetching Wikipedia tables from {url}")
tables = pd.read_html(url, flavor="lxml") req = Request(url, headers={"User-Agent": "Mozilla/5.0 (compatible; StockOracle/1.0)"})
with urlopen(req, timeout=25) as resp:
html = resp.read().decode("utf-8")
tables = pd.read_html(StringIO(html), flavor="lxml")
# Try configured table index first # Try configured table index first
df = None df = None

Loading…
Cancel
Save