You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
454 lines
20 KiB
Python
454 lines
20 KiB
Python
"""
|
|
Shared SEC EDGAR HTTP client with retry, backoff, throttling, and caching.
|
|
|
|
Extracted from etf_holdings_fetcher.py to be reused by all SEC-related services.
|
|
"""
|
|
|
|
import time as _time
|
|
import asyncio
|
|
import contextvars
|
|
import aiohttp
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import random
|
|
import logging
|
|
from typing import Dict, Optional
|
|
|
|
from app.core.config import settings
|
|
|
|
# Per-asyncio-task deadline — isolates concurrent requests from each other
|
|
_deadline_var: contextvars.ContextVar[Optional[float]] = contextvars.ContextVar(
|
|
"sec_http_deadline", default=None
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Only cache small responses in memory; large responses go to disk cache only
|
|
_MAX_MEMORY_CACHE_BYTES = 1 * 1024 * 1024 # 1MB
|
|
|
|
|
|
def _is_sec_block_page(text: str) -> bool:
|
|
if not text:
|
|
return False
|
|
tl = text.lower()
|
|
if "your request originates from an undeclared automated tool" in tl:
|
|
return True
|
|
if "<title>sec.gov | your request originates" in tl:
|
|
return True
|
|
if "reference id:" in tl and "sec.gov" in tl:
|
|
return True
|
|
return False
|
|
|
|
|
|
class _TokenBucket:
|
|
"""Async token bucket rate limiter.
|
|
|
|
Allows up to `rate` requests per second with a burst capacity of `capacity`.
|
|
"""
|
|
|
|
def __init__(self, rate: float, capacity: float) -> None:
|
|
self._rate = rate
|
|
self._capacity = capacity
|
|
self._tokens = capacity
|
|
self._last_refill = _time.monotonic()
|
|
self._lock = asyncio.Lock()
|
|
|
|
async def acquire(self) -> None:
|
|
while True:
|
|
async with self._lock:
|
|
now = _time.monotonic()
|
|
elapsed = now - self._last_refill
|
|
self._tokens = min(self._capacity, self._tokens + elapsed * self._rate)
|
|
self._last_refill = now
|
|
if self._tokens >= 1.0:
|
|
self._tokens -= 1.0
|
|
return
|
|
wait_time = (1.0 - self._tokens) / self._rate
|
|
await asyncio.sleep(wait_time)
|
|
|
|
|
|
class SECHttpClient:
|
|
"""Shared HTTP client for SEC EDGAR API requests.
|
|
|
|
Features:
|
|
- asyncio.Semaphore(2) concurrent request limit
|
|
- Token bucket rate limiter (10 req/sec)
|
|
- 6 retries with 1.8x exponential backoff + random jitter
|
|
- 429 Retry-After header respect
|
|
- SEC block page detection
|
|
- Deadline-aware timeouts
|
|
- Disk cache (/tmp/stock_oracle_sec_cache/, SHA-256 keyed)
|
|
- In-memory cache (_json_cache, _text_cache)
|
|
"""
|
|
|
|
# Max total pending (waiting + active) SEC HTTP requests across all callers.
|
|
# Requests beyond this limit are rejected immediately to protect the event loop.
|
|
_MAX_PENDING: int = 50
|
|
|
|
def __init__(self, user_agent_name: str = "Stock Oracle"):
|
|
self.sec_base_data = "https://data.sec.gov"
|
|
self.sec_base_www = "https://www.sec.gov"
|
|
self.http_timeout = aiohttp.ClientTimeout(total=12)
|
|
self._req_sem = asyncio.Semaphore(2)
|
|
self._rate_limiter = _TokenBucket(rate=8.0, capacity=8.0)
|
|
self._pending: int = 0
|
|
self._text_cache: Dict[str, str] = {}
|
|
self._json_cache: Dict[str, dict] = {}
|
|
self._cache_dir = "/tmp/stock_oracle_sec_cache"
|
|
try:
|
|
os.makedirs(self._cache_dir, exist_ok=True)
|
|
except Exception:
|
|
pass
|
|
self._user_agent = f"{user_agent_name} ({settings.SEC_EMAIL})"
|
|
self._session: Optional[aiohttp.ClientSession] = None
|
|
|
|
async def _get_session(self) -> aiohttp.ClientSession:
|
|
"""Get or create a persistent session."""
|
|
if self._session is None or self._session.closed:
|
|
self._session = aiohttp.ClientSession(
|
|
timeout=self.http_timeout,
|
|
headers={"User-Agent": self._user_agent},
|
|
)
|
|
return self._session
|
|
|
|
async def close(self) -> None:
|
|
"""Close the persistent session."""
|
|
if self._session and not self._session.closed:
|
|
await self._session.close()
|
|
self._session = None
|
|
|
|
# ------------------------------------------------------------------
|
|
# Deadline management
|
|
# ------------------------------------------------------------------
|
|
|
|
def set_deadline(self, seconds_from_now: float) -> None:
|
|
_deadline_var.set(_time.monotonic() + seconds_from_now)
|
|
|
|
def clear_deadline(self) -> None:
|
|
_deadline_var.set(None)
|
|
|
|
@property
|
|
def deadline(self) -> Optional[float]:
|
|
return _deadline_var.get()
|
|
|
|
@deadline.setter
|
|
def deadline(self, value: Optional[float]) -> None:
|
|
_deadline_var.set(value)
|
|
|
|
def remaining_time(self) -> Optional[float]:
|
|
dl = _deadline_var.get()
|
|
if dl is None:
|
|
return None
|
|
return max(0.0, dl - _time.monotonic())
|
|
|
|
def is_deadline_exceeded(self) -> bool:
|
|
dl = _deadline_var.get()
|
|
if dl is None:
|
|
return False
|
|
return _time.monotonic() >= dl
|
|
|
|
# ------------------------------------------------------------------
|
|
# Cache helpers
|
|
# ------------------------------------------------------------------
|
|
|
|
def _cache_path(self, url: str) -> str:
|
|
h = hashlib.sha256(url.encode("utf-8")).hexdigest()
|
|
return os.path.join(self._cache_dir, h)
|
|
|
|
# ------------------------------------------------------------------
|
|
# CIK lookup
|
|
# ------------------------------------------------------------------
|
|
|
|
async def get_company_cik(self, ticker: str) -> Optional[str]:
|
|
"""Look up zero-padded 10-digit CIK for a ticker."""
|
|
url = f"{self.sec_base_www}/files/company_tickers.json"
|
|
try:
|
|
data = await self.fetch_json(url)
|
|
for _key, company_info in data.items():
|
|
if company_info.get("ticker", "").upper() == ticker.upper():
|
|
cik_str = str(company_info.get("cik_str", "")).zfill(10)
|
|
logger.info(f"Found CIK {cik_str} for ticker {ticker}")
|
|
return cik_str
|
|
logger.warning(f"Ticker {ticker} not found in SEC mapping")
|
|
return None
|
|
except Exception as e:
|
|
logger.error(f"Error fetching CIK for {ticker}: {e}")
|
|
return None
|
|
|
|
# ------------------------------------------------------------------
|
|
# HTTP fetch methods
|
|
# ------------------------------------------------------------------
|
|
|
|
async def fetch_json(self, url: str) -> dict:
|
|
"""Fetch JSON with retry, backoff, and caching."""
|
|
# In-memory cache
|
|
if url in self._json_cache:
|
|
return self._json_cache[url]
|
|
# Disk cache
|
|
try:
|
|
cp = self._cache_path(url) + ".json"
|
|
if os.path.exists(cp):
|
|
ttl_sec = max(3600, settings.SEC_DATA_REFRESH_HOURS * 3600)
|
|
if _time.time() - os.path.getmtime(cp) <= ttl_sec:
|
|
with open(cp, "r", encoding="utf-8") as f:
|
|
data = json.load(f)
|
|
self._json_cache[url] = data
|
|
return data
|
|
except Exception:
|
|
pass
|
|
|
|
# Backpressure: reject immediately if too many SEC requests are already pending.
|
|
# This prevents thousands of coroutines from stacking up in the event loop,
|
|
# which would starve health checks and other endpoints.
|
|
if self._pending >= self._MAX_PENDING:
|
|
raise RuntimeError(
|
|
f"SEC request queue full ({self._pending}/{self._MAX_PENDING} pending)"
|
|
)
|
|
self._pending += 1
|
|
try:
|
|
attempts = 6
|
|
backoff = 1.0
|
|
last_exc = None
|
|
for _i in range(attempts):
|
|
now = _time.monotonic()
|
|
dl = _deadline_var.get()
|
|
if dl is not None and now >= dl:
|
|
break
|
|
req_timeout = self.http_timeout
|
|
if dl is not None:
|
|
remaining = max(0.0, dl - now)
|
|
if remaining < 0.25:
|
|
break
|
|
req_timeout = aiohttp.ClientTimeout(
|
|
total=min(remaining, getattr(self.http_timeout, "total", 12))
|
|
)
|
|
async with self._req_sem:
|
|
await self._rate_limiter.acquire()
|
|
try:
|
|
session = await self._get_session()
|
|
async with session.get(url, timeout=req_timeout, headers={"Accept": "application/json"}) as resp:
|
|
if resp.status == 429:
|
|
retry_after = resp.headers.get("Retry-After")
|
|
delay = (
|
|
float(retry_after)
|
|
if retry_after and retry_after.isdigit()
|
|
else backoff
|
|
)
|
|
dl = _deadline_var.get()
|
|
if dl is not None:
|
|
remaining = max(0.0, dl - _time.monotonic())
|
|
delay = min(delay, max(0.0, remaining - 0.05))
|
|
await asyncio.sleep(
|
|
max(0.0, delay)
|
|
+ random.uniform(0.0, delay * 0.25 if delay > 0 else 0.0)
|
|
)
|
|
backoff *= 1.8
|
|
continue
|
|
if 500 <= resp.status < 600:
|
|
delay = backoff
|
|
dl = _deadline_var.get()
|
|
if dl is not None:
|
|
remaining = max(0.0, dl - _time.monotonic())
|
|
delay = min(delay, max(0.0, remaining - 0.05))
|
|
await asyncio.sleep(
|
|
max(0.0, delay)
|
|
+ random.uniform(0.0, delay * 0.25 if delay > 0 else 0.0)
|
|
)
|
|
backoff *= 1.8
|
|
continue
|
|
resp.raise_for_status()
|
|
data = await resp.json()
|
|
# Cache successful response
|
|
try:
|
|
with open(self._cache_path(url) + ".json", "w", encoding="utf-8") as f:
|
|
json.dump(data, f)
|
|
except Exception:
|
|
pass
|
|
self._json_cache[url] = data
|
|
return data
|
|
except aiohttp.ClientResponseError as e:
|
|
if 400 <= e.status < 500:
|
|
raise # 4xx: no retry, raise immediately
|
|
last_exc = e
|
|
delay = backoff
|
|
dl = _deadline_var.get()
|
|
if dl is not None:
|
|
remaining = max(0.0, dl - _time.monotonic())
|
|
delay = min(delay, max(0.0, remaining - 0.05))
|
|
await asyncio.sleep(
|
|
max(0.0, delay)
|
|
+ random.uniform(0.0, delay * 0.25 if delay > 0 else 0.0)
|
|
)
|
|
backoff *= 1.8
|
|
continue
|
|
except Exception as e:
|
|
last_exc = e
|
|
delay = backoff
|
|
dl = _deadline_var.get()
|
|
if dl is not None:
|
|
remaining = max(0.0, dl - _time.monotonic())
|
|
delay = min(delay, max(0.0, remaining - 0.05))
|
|
await asyncio.sleep(
|
|
max(0.0, delay)
|
|
+ random.uniform(0.0, delay * 0.25 if delay > 0 else 0.0)
|
|
)
|
|
backoff *= 1.8
|
|
continue
|
|
raise last_exc if last_exc else RuntimeError("Failed to fetch JSON")
|
|
finally:
|
|
self._pending -= 1
|
|
|
|
async def fetch_text(self, url: str, accept: str = "text/html", max_bytes: Optional[int] = None) -> str:
|
|
"""Fetch text content with retry, backoff, block page detection, and caching."""
|
|
# In-memory cache
|
|
if url in self._text_cache:
|
|
return self._text_cache[url]
|
|
# Disk cache
|
|
try:
|
|
cp = self._cache_path(url) + ".txt"
|
|
if os.path.exists(cp):
|
|
ttl_sec = max(3600, settings.SEC_DATA_REFRESH_HOURS * 3600)
|
|
if _time.time() - os.path.getmtime(cp) <= ttl_sec:
|
|
with open(cp, "r", encoding="utf-8") as f:
|
|
text = f.read()
|
|
if _is_sec_block_page(text):
|
|
try:
|
|
os.remove(cp)
|
|
except Exception:
|
|
pass
|
|
else:
|
|
if len(text) <= _MAX_MEMORY_CACHE_BYTES:
|
|
self._text_cache[url] = text
|
|
return text
|
|
except Exception:
|
|
pass
|
|
|
|
# Backpressure: reject immediately if too many SEC requests are already pending.
|
|
if self._pending >= self._MAX_PENDING:
|
|
raise RuntimeError(
|
|
f"SEC request queue full ({self._pending}/{self._MAX_PENDING} pending)"
|
|
)
|
|
self._pending += 1
|
|
try:
|
|
attempts = 6
|
|
backoff = 1.0
|
|
last_exc = None
|
|
for _i in range(attempts):
|
|
now = _time.monotonic()
|
|
dl = _deadline_var.get()
|
|
if dl is not None and now >= dl:
|
|
break
|
|
req_timeout = self.http_timeout
|
|
if dl is not None:
|
|
remaining = max(0.0, dl - now)
|
|
if remaining < 0.25:
|
|
break
|
|
req_timeout = aiohttp.ClientTimeout(
|
|
total=min(remaining, getattr(self.http_timeout, "total", 12))
|
|
)
|
|
async with self._req_sem:
|
|
await self._rate_limiter.acquire()
|
|
try:
|
|
session = await self._get_session()
|
|
async with session.get(url, timeout=req_timeout, headers={"Accept": accept}) as resp:
|
|
if resp.status == 429:
|
|
retry_after = resp.headers.get("Retry-After")
|
|
delay = (
|
|
float(retry_after)
|
|
if retry_after and retry_after.isdigit()
|
|
else backoff
|
|
)
|
|
dl = _deadline_var.get()
|
|
if dl is not None:
|
|
remaining = max(0.0, dl - _time.monotonic())
|
|
delay = min(delay, max(0.0, remaining - 0.05))
|
|
await asyncio.sleep(
|
|
max(0.0, delay)
|
|
+ random.uniform(0.0, delay * 0.25 if delay > 0 else 0.0)
|
|
)
|
|
backoff *= 1.8
|
|
continue
|
|
if 500 <= resp.status < 600:
|
|
delay = backoff
|
|
dl = _deadline_var.get()
|
|
if dl is not None:
|
|
remaining = max(0.0, dl - _time.monotonic())
|
|
delay = min(delay, max(0.0, remaining - 0.05))
|
|
await asyncio.sleep(
|
|
max(0.0, delay)
|
|
+ random.uniform(0.0, delay * 0.25 if delay > 0 else 0.0)
|
|
)
|
|
backoff *= 1.8
|
|
continue
|
|
resp.raise_for_status()
|
|
if max_bytes is not None:
|
|
cl = resp.content_length
|
|
if cl is not None and cl > max_bytes:
|
|
raise ValueError(
|
|
f"Response too large ({cl} bytes > {max_bytes} limit)"
|
|
)
|
|
raw = await resp.content.read(max_bytes + 1)
|
|
if len(raw) > max_bytes:
|
|
raise ValueError(
|
|
f"Response exceeded {max_bytes} byte limit"
|
|
)
|
|
text = raw.decode(resp.get_encoding() or "utf-8", errors="replace")
|
|
else:
|
|
text = await resp.text()
|
|
if _is_sec_block_page(text):
|
|
last_exc = RuntimeError("SEC_BLOCKED")
|
|
delay = backoff * 2.0
|
|
dl = _deadline_var.get()
|
|
if dl is not None:
|
|
remaining = max(0.0, dl - _time.monotonic())
|
|
delay = min(delay, max(0.0, remaining - 0.05))
|
|
await asyncio.sleep(
|
|
max(0.0, delay)
|
|
+ random.uniform(0.0, delay * 0.5 if delay > 0 else 0.0)
|
|
)
|
|
backoff *= 2.0
|
|
continue
|
|
# Cache successful response
|
|
if len(text) <= _MAX_MEMORY_CACHE_BYTES:
|
|
self._text_cache[url] = text
|
|
try:
|
|
with open(self._cache_path(url) + ".txt", "w", encoding="utf-8") as f:
|
|
f.write(text)
|
|
except Exception:
|
|
pass
|
|
return text
|
|
except aiohttp.ClientResponseError as e:
|
|
if 400 <= e.status < 500:
|
|
raise # 4xx: no retry, raise immediately
|
|
last_exc = e
|
|
delay = backoff
|
|
dl = _deadline_var.get()
|
|
if dl is not None:
|
|
remaining = max(0.0, dl - _time.monotonic())
|
|
delay = min(delay, max(0.0, remaining - 0.05))
|
|
await asyncio.sleep(
|
|
max(0.0, delay)
|
|
+ random.uniform(0.0, delay * 0.25 if delay > 0 else 0.0)
|
|
)
|
|
backoff *= 1.8
|
|
continue
|
|
except ValueError:
|
|
raise # Size limit exceeded: no retry (size won't change)
|
|
except Exception as e:
|
|
last_exc = e
|
|
delay = backoff
|
|
dl = _deadline_var.get()
|
|
if dl is not None:
|
|
remaining = max(0.0, dl - _time.monotonic())
|
|
delay = min(delay, max(0.0, remaining - 0.05))
|
|
await asyncio.sleep(
|
|
max(0.0, delay)
|
|
+ random.uniform(0.0, delay * 0.25 if delay > 0 else 0.0)
|
|
)
|
|
backoff *= 1.8
|
|
continue
|
|
raise last_exc if last_exc else RuntimeError("Failed to fetch text")
|
|
finally:
|
|
self._pending -= 1
|