""" StockTwits public API client. Endpoint: GET /streams/symbol/{ticker}.json on api.stocktwits.com/api/2 Auth: none required for public streams. Rate limit: ~200 req/hr per IP. We enforce a token bucket of one request per 3.6 seconds (60 / hour buffer below the 200/hr ceiling) — well under the documented limit even with multiple ingestor processes. """ from __future__ import annotations import asyncio import logging import time from typing import Any import httpx from app.core.config import settings logger = logging.getLogger(__name__) class StocktwitsClient: def __init__( self, base_url: str | None = None, request_interval_sec: float = 3.6, ): self.base_url = (base_url or settings.STOCKTWITS_BASE_URL).rstrip("/") self._client: httpx.AsyncClient | None = None self._gate = asyncio.Semaphore(1) self._interval = request_interval_sec self._last_request_at: float = 0.0 def is_configured(self) -> bool: return True # public API async def _get_client(self) -> httpx.AsyncClient: if self._client is None or self._client.is_closed: self._client = httpx.AsyncClient(base_url=self.base_url, timeout=20.0) return self._client async def close(self) -> None: if self._client and not self._client.is_closed: await self._client.aclose() async def fetch_symbol_stream( self, symbol: str, since_id: int | None = None, max_results: int = 30, ) -> dict[str, Any]: """ Fetch the recent message stream for one symbol. `since_id` (StockTwits parameter `since`) yields only messages whose ID is greater than the given value. Pass the highest message ID seen on the previous poll to stay incremental and avoid re-ingesting duplicates (dedup is also enforced at the DB layer via uq_news_headline_*). """ params: dict[str, Any] = {"limit": min(max_results, 30)} if since_id is not None: params["since"] = since_id sym = symbol.strip().upper() path = f"/streams/symbol/{sym}.json" async with self._gate: now = time.monotonic() wait = self._interval - (now - self._last_request_at) if wait > 0: await asyncio.sleep(wait) client = await self._get_client() try: resp = await client.get(path, params=params) self._last_request_at = time.monotonic() except (httpx.ConnectError, httpx.ReadTimeout) as e: logger.warning(f"StockTwits fetch failed {sym}: {e}") return {"messages": []} if resp.status_code == 429: logger.warning(f"StockTwits 429 on {sym} — caller should back off") return {"messages": []} if resp.status_code == 404: # Symbol not found on StockTwits — treat as empty rather than error return {"messages": []} try: resp.raise_for_status() except httpx.HTTPStatusError as e: logger.warning(f"StockTwits {resp.status_code} on {sym}: {e}") return {"messages": []} return resp.json()