You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
97 lines
3.2 KiB
Python
97 lines
3.2 KiB
Python
"""
|
|
StockTwits public API client.
|
|
|
|
Endpoint: GET /streams/symbol/{ticker}.json on api.stocktwits.com/api/2
|
|
Auth: none required for public streams.
|
|
Rate limit: ~200 req/hr per IP. We enforce a token bucket of one request per
|
|
3.6 seconds (60 / hour buffer below the 200/hr ceiling) — well under the
|
|
documented limit even with multiple ingestor processes.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import time
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from app.core.config import settings
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class StocktwitsClient:
|
|
def __init__(
|
|
self,
|
|
base_url: str | None = None,
|
|
request_interval_sec: float = 3.6,
|
|
):
|
|
self.base_url = (base_url or settings.STOCKTWITS_BASE_URL).rstrip("/")
|
|
self._client: httpx.AsyncClient | None = None
|
|
self._gate = asyncio.Semaphore(1)
|
|
self._interval = request_interval_sec
|
|
self._last_request_at: float = 0.0
|
|
|
|
def is_configured(self) -> bool:
|
|
return True # public API
|
|
|
|
async def _get_client(self) -> httpx.AsyncClient:
|
|
if self._client is None or self._client.is_closed:
|
|
self._client = httpx.AsyncClient(base_url=self.base_url, timeout=20.0)
|
|
return self._client
|
|
|
|
async def close(self) -> None:
|
|
if self._client and not self._client.is_closed:
|
|
await self._client.aclose()
|
|
|
|
async def fetch_symbol_stream(
|
|
self,
|
|
symbol: str,
|
|
since_id: int | None = None,
|
|
max_results: int = 30,
|
|
) -> dict[str, Any]:
|
|
"""
|
|
Fetch the recent message stream for one symbol.
|
|
|
|
`since_id` (StockTwits parameter `since`) yields only messages whose ID
|
|
is greater than the given value. Pass the highest message ID seen on
|
|
the previous poll to stay incremental and avoid re-ingesting duplicates
|
|
(dedup is also enforced at the DB layer via uq_news_headline_*).
|
|
"""
|
|
params: dict[str, Any] = {"limit": min(max_results, 30)}
|
|
if since_id is not None:
|
|
params["since"] = since_id
|
|
|
|
sym = symbol.strip().upper()
|
|
path = f"/streams/symbol/{sym}.json"
|
|
|
|
async with self._gate:
|
|
now = time.monotonic()
|
|
wait = self._interval - (now - self._last_request_at)
|
|
if wait > 0:
|
|
await asyncio.sleep(wait)
|
|
|
|
client = await self._get_client()
|
|
try:
|
|
resp = await client.get(path, params=params)
|
|
self._last_request_at = time.monotonic()
|
|
except (httpx.ConnectError, httpx.ReadTimeout) as e:
|
|
logger.warning(f"StockTwits fetch failed {sym}: {e}")
|
|
return {"messages": []}
|
|
|
|
if resp.status_code == 429:
|
|
logger.warning(f"StockTwits 429 on {sym} — caller should back off")
|
|
return {"messages": []}
|
|
if resp.status_code == 404:
|
|
# Symbol not found on StockTwits — treat as empty rather than error
|
|
return {"messages": []}
|
|
try:
|
|
resp.raise_for_status()
|
|
except httpx.HTTPStatusError as e:
|
|
logger.warning(f"StockTwits {resp.status_code} on {sym}: {e}")
|
|
return {"messages": []}
|
|
|
|
return resp.json()
|