You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
328 lines
14 KiB
Python
328 lines
14 KiB
Python
"""
|
|
Shared SEC EDGAR HTTP client with retry, backoff, throttling, and caching.
|
|
|
|
Extracted from etf_holdings_fetcher.py to be reused by all SEC-related services.
|
|
"""
|
|
|
|
import time as _time
|
|
import asyncio
|
|
import aiohttp
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import random
|
|
import logging
|
|
from typing import Dict, Optional
|
|
|
|
from app.core.config import settings
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def _is_sec_block_page(text: str) -> bool:
|
|
if not text:
|
|
return False
|
|
tl = text.lower()
|
|
if "your request originates from an undeclared automated tool" in tl:
|
|
return True
|
|
if "<title>sec.gov | your request originates" in tl:
|
|
return True
|
|
if "reference id:" in tl and "sec.gov" in tl:
|
|
return True
|
|
return False
|
|
|
|
|
|
class SECHttpClient:
|
|
"""Shared HTTP client for SEC EDGAR API requests.
|
|
|
|
Features:
|
|
- asyncio.Semaphore(2) concurrent request limit
|
|
- 6 retries with 1.8x exponential backoff + random jitter
|
|
- 429 Retry-After header respect
|
|
- SEC block page detection
|
|
- Deadline-aware timeouts
|
|
- Disk cache (/tmp/stock_oracle_sec_cache/, SHA-256 keyed)
|
|
- In-memory cache (_json_cache, _text_cache)
|
|
"""
|
|
|
|
def __init__(self, user_agent_name: str = "Stock Oracle"):
|
|
self.sec_base_data = "https://data.sec.gov"
|
|
self.sec_base_www = "https://www.sec.gov"
|
|
self.http_timeout = aiohttp.ClientTimeout(total=12)
|
|
self._req_sem = asyncio.Semaphore(2)
|
|
self._text_cache: Dict[str, str] = {}
|
|
self._json_cache: Dict[str, dict] = {}
|
|
self._cache_dir = "/tmp/stock_oracle_sec_cache"
|
|
try:
|
|
os.makedirs(self._cache_dir, exist_ok=True)
|
|
except Exception:
|
|
pass
|
|
self._user_agent = f"{user_agent_name} ({settings.SEC_EMAIL})"
|
|
self._deadline: Optional[float] = None
|
|
|
|
# ------------------------------------------------------------------
|
|
# Deadline management
|
|
# ------------------------------------------------------------------
|
|
|
|
def set_deadline(self, seconds_from_now: float) -> None:
|
|
self._deadline = _time.monotonic() + seconds_from_now
|
|
|
|
def clear_deadline(self) -> None:
|
|
self._deadline = None
|
|
|
|
@property
|
|
def deadline(self) -> Optional[float]:
|
|
return self._deadline
|
|
|
|
@deadline.setter
|
|
def deadline(self, value: Optional[float]) -> None:
|
|
self._deadline = value
|
|
|
|
def remaining_time(self) -> Optional[float]:
|
|
if self._deadline is None:
|
|
return None
|
|
return max(0.0, self._deadline - _time.monotonic())
|
|
|
|
def is_deadline_exceeded(self) -> bool:
|
|
if self._deadline is None:
|
|
return False
|
|
return _time.monotonic() >= self._deadline
|
|
|
|
# ------------------------------------------------------------------
|
|
# Cache helpers
|
|
# ------------------------------------------------------------------
|
|
|
|
def _cache_path(self, url: str) -> str:
|
|
h = hashlib.sha256(url.encode("utf-8")).hexdigest()
|
|
return os.path.join(self._cache_dir, h)
|
|
|
|
# ------------------------------------------------------------------
|
|
# CIK lookup
|
|
# ------------------------------------------------------------------
|
|
|
|
async def get_company_cik(self, ticker: str) -> Optional[str]:
|
|
"""Look up zero-padded 10-digit CIK for a ticker."""
|
|
url = f"{self.sec_base_www}/files/company_tickers.json"
|
|
try:
|
|
data = await self.fetch_json(url)
|
|
for _key, company_info in data.items():
|
|
if company_info.get("ticker", "").upper() == ticker.upper():
|
|
cik_str = str(company_info.get("cik_str", "")).zfill(10)
|
|
logger.info(f"Found CIK {cik_str} for ticker {ticker}")
|
|
return cik_str
|
|
logger.warning(f"Ticker {ticker} not found in SEC mapping")
|
|
return None
|
|
except Exception as e:
|
|
logger.error(f"Error fetching CIK for {ticker}: {e}")
|
|
return None
|
|
|
|
# ------------------------------------------------------------------
|
|
# HTTP fetch methods
|
|
# ------------------------------------------------------------------
|
|
|
|
async def fetch_json(self, url: str) -> dict:
|
|
"""Fetch JSON with retry, backoff, and caching."""
|
|
# In-memory cache
|
|
if url in self._json_cache:
|
|
return self._json_cache[url]
|
|
# Disk cache
|
|
try:
|
|
cp = self._cache_path(url) + ".json"
|
|
if os.path.exists(cp):
|
|
ttl_sec = max(3600, settings.SEC_DATA_REFRESH_HOURS * 3600)
|
|
if _time.time() - os.path.getmtime(cp) <= ttl_sec:
|
|
with open(cp, "r", encoding="utf-8") as f:
|
|
data = json.load(f)
|
|
self._json_cache[url] = data
|
|
return data
|
|
except Exception:
|
|
pass
|
|
|
|
attempts = 6
|
|
backoff = 1.0
|
|
last_exc = None
|
|
for _i in range(attempts):
|
|
now = _time.monotonic()
|
|
if self._deadline is not None and now >= self._deadline:
|
|
break
|
|
req_timeout = self.http_timeout
|
|
if self._deadline is not None:
|
|
remaining = max(0.0, self._deadline - now)
|
|
if remaining < 0.25:
|
|
break
|
|
req_timeout = aiohttp.ClientTimeout(
|
|
total=min(remaining, getattr(self.http_timeout, "total", 12))
|
|
)
|
|
async with self._req_sem:
|
|
try:
|
|
async with aiohttp.ClientSession(
|
|
timeout=req_timeout,
|
|
headers={
|
|
"User-Agent": self._user_agent,
|
|
"Accept": "application/json",
|
|
},
|
|
) as session:
|
|
async with session.get(url, timeout=req_timeout) as resp:
|
|
if resp.status == 429:
|
|
retry_after = resp.headers.get("Retry-After")
|
|
delay = (
|
|
float(retry_after)
|
|
if retry_after and retry_after.isdigit()
|
|
else backoff
|
|
)
|
|
if self._deadline is not None:
|
|
remaining = max(0.0, self._deadline - _time.monotonic())
|
|
delay = min(delay, max(0.0, remaining - 0.05))
|
|
await asyncio.sleep(
|
|
max(0.0, delay)
|
|
+ random.uniform(0.0, delay * 0.25 if delay > 0 else 0.0)
|
|
)
|
|
backoff *= 1.8
|
|
continue
|
|
if 500 <= resp.status < 600:
|
|
delay = backoff
|
|
if self._deadline is not None:
|
|
remaining = max(0.0, self._deadline - _time.monotonic())
|
|
delay = min(delay, max(0.0, remaining - 0.05))
|
|
await asyncio.sleep(
|
|
max(0.0, delay)
|
|
+ random.uniform(0.0, delay * 0.25 if delay > 0 else 0.0)
|
|
)
|
|
backoff *= 1.8
|
|
continue
|
|
resp.raise_for_status()
|
|
data = await resp.json()
|
|
# Cache successful response
|
|
try:
|
|
with open(self._cache_path(url) + ".json", "w", encoding="utf-8") as f:
|
|
json.dump(data, f)
|
|
except Exception:
|
|
pass
|
|
self._json_cache[url] = data
|
|
return data
|
|
except Exception as e:
|
|
last_exc = e
|
|
delay = backoff
|
|
if self._deadline is not None:
|
|
remaining = max(0.0, self._deadline - _time.monotonic())
|
|
delay = min(delay, max(0.0, remaining - 0.05))
|
|
await asyncio.sleep(
|
|
max(0.0, delay)
|
|
+ random.uniform(0.0, delay * 0.25 if delay > 0 else 0.0)
|
|
)
|
|
backoff *= 1.8
|
|
continue
|
|
raise last_exc if last_exc else RuntimeError("Failed to fetch JSON")
|
|
|
|
async def fetch_text(self, url: str, accept: str = "text/html") -> str:
|
|
"""Fetch text content with retry, backoff, block page detection, and caching."""
|
|
# In-memory cache
|
|
if url in self._text_cache:
|
|
return self._text_cache[url]
|
|
# Disk cache
|
|
try:
|
|
cp = self._cache_path(url) + ".txt"
|
|
if os.path.exists(cp):
|
|
ttl_sec = max(3600, settings.SEC_DATA_REFRESH_HOURS * 3600)
|
|
if _time.time() - os.path.getmtime(cp) <= ttl_sec:
|
|
with open(cp, "r", encoding="utf-8") as f:
|
|
text = f.read()
|
|
if _is_sec_block_page(text):
|
|
try:
|
|
os.remove(cp)
|
|
except Exception:
|
|
pass
|
|
else:
|
|
self._text_cache[url] = text
|
|
return text
|
|
except Exception:
|
|
pass
|
|
|
|
attempts = 6
|
|
backoff = 1.0
|
|
last_exc = None
|
|
for _i in range(attempts):
|
|
now = _time.monotonic()
|
|
if self._deadline is not None and now >= self._deadline:
|
|
break
|
|
req_timeout = self.http_timeout
|
|
if self._deadline is not None:
|
|
remaining = max(0.0, self._deadline - now)
|
|
if remaining < 0.25:
|
|
break
|
|
req_timeout = aiohttp.ClientTimeout(
|
|
total=min(remaining, getattr(self.http_timeout, "total", 12))
|
|
)
|
|
async with self._req_sem:
|
|
try:
|
|
async with aiohttp.ClientSession(
|
|
timeout=req_timeout,
|
|
headers={
|
|
"User-Agent": self._user_agent,
|
|
"Accept": accept,
|
|
},
|
|
) as session:
|
|
async with session.get(url, timeout=req_timeout) as resp:
|
|
if resp.status == 429:
|
|
retry_after = resp.headers.get("Retry-After")
|
|
delay = (
|
|
float(retry_after)
|
|
if retry_after and retry_after.isdigit()
|
|
else backoff
|
|
)
|
|
if self._deadline is not None:
|
|
remaining = max(0.0, self._deadline - _time.monotonic())
|
|
delay = min(delay, max(0.0, remaining - 0.05))
|
|
await asyncio.sleep(
|
|
max(0.0, delay)
|
|
+ random.uniform(0.0, delay * 0.25 if delay > 0 else 0.0)
|
|
)
|
|
backoff *= 1.8
|
|
continue
|
|
if 500 <= resp.status < 600:
|
|
delay = backoff
|
|
if self._deadline is not None:
|
|
remaining = max(0.0, self._deadline - _time.monotonic())
|
|
delay = min(delay, max(0.0, remaining - 0.05))
|
|
await asyncio.sleep(
|
|
max(0.0, delay)
|
|
+ random.uniform(0.0, delay * 0.25 if delay > 0 else 0.0)
|
|
)
|
|
backoff *= 1.8
|
|
continue
|
|
resp.raise_for_status()
|
|
text = await resp.text()
|
|
if _is_sec_block_page(text):
|
|
last_exc = RuntimeError("SEC_BLOCKED")
|
|
delay = backoff * 2.0
|
|
if self._deadline is not None:
|
|
remaining = max(0.0, self._deadline - _time.monotonic())
|
|
delay = min(delay, max(0.0, remaining - 0.05))
|
|
await asyncio.sleep(
|
|
max(0.0, delay)
|
|
+ random.uniform(0.0, delay * 0.5 if delay > 0 else 0.0)
|
|
)
|
|
backoff *= 2.0
|
|
continue
|
|
# Cache successful response
|
|
self._text_cache[url] = text
|
|
try:
|
|
with open(self._cache_path(url) + ".txt", "w", encoding="utf-8") as f:
|
|
f.write(text)
|
|
except Exception:
|
|
pass
|
|
return text
|
|
except Exception as e:
|
|
last_exc = e
|
|
delay = backoff
|
|
if self._deadline is not None:
|
|
remaining = max(0.0, self._deadline - _time.monotonic())
|
|
delay = min(delay, max(0.0, remaining - 0.05))
|
|
await asyncio.sleep(
|
|
max(0.0, delay)
|
|
+ random.uniform(0.0, delay * 0.25 if delay > 0 else 0.0)
|
|
)
|
|
backoff *= 1.8
|
|
continue
|
|
raise last_exc if last_exc else RuntimeError("Failed to fetch text")
|