"""
Shared SEC EDGAR HTTP client with retry, backoff, throttling, and caching.

Extracted from etf_holdings_fetcher.py to be reused by all SEC-related services.
"""

import time as _time
import asyncio
import contextvars
import aiohttp
import hashlib
import json
import os
import random
import logging
from typing import Dict, Optional

from app.core.config import settings

# Per-asyncio-task deadline — isolates concurrent requests from each other
_deadline_var: contextvars.ContextVar[Optional[float]] = contextvars.ContextVar(
    "sec_http_deadline", default=None
)

logger = logging.getLogger(__name__)

# Only cache small responses in memory; large responses go to disk cache only
_MAX_MEMORY_CACHE_BYTES = 1 * 1024 * 1024  # 1MB


def _is_sec_block_page(text: str) -> bool:
    if not text:
        return False
    tl = text.lower()
    if "your request originates from an undeclared automated tool" in tl:
        return True
    if "<title>sec.gov | your request originates" in tl:
        return True
    if "reference id:" in tl and "sec.gov" in tl:
        return True
    return False


class _TokenBucket:
    """Async token bucket rate limiter.

    Allows up to `rate` requests per second with a burst capacity of `capacity`.
    """

    def __init__(self, rate: float, capacity: float) -> None:
        self._rate = rate
        self._capacity = capacity
        self._tokens = capacity
        self._last_refill = _time.monotonic()
        self._lock = asyncio.Lock()

    async def acquire(self) -> None:
        while True:
            async with self._lock:
                now = _time.monotonic()
                elapsed = now - self._last_refill
                self._tokens = min(self._capacity, self._tokens + elapsed * self._rate)
                self._last_refill = now
                if self._tokens >= 1.0:
                    self._tokens -= 1.0
                    return
                wait_time = (1.0 - self._tokens) / self._rate
            await asyncio.sleep(wait_time)


class SECHttpClient:
    """Shared HTTP client for SEC EDGAR API requests.

    Features:
    - asyncio.Semaphore(2) concurrent request limit
    - Token bucket rate limiter (10 req/sec)
    - 6 retries with 1.8x exponential backoff + random jitter
    - 429 Retry-After header respect
    - SEC block page detection
    - Deadline-aware timeouts
    - Disk cache (/tmp/stock_oracle_sec_cache/, SHA-256 keyed)
    - In-memory cache (_json_cache, _text_cache)
    """

    # Max total pending (waiting + active) SEC HTTP requests across all callers.
    # Requests beyond this limit are rejected immediately to protect the event loop.
    _MAX_PENDING: int = 50

    def __init__(self, user_agent_name: str = "Stock Oracle"):
        self.sec_base_data = "https://data.sec.gov"
        self.sec_base_www = "https://www.sec.gov"
        self.http_timeout = aiohttp.ClientTimeout(total=12)
        self._req_sem = asyncio.Semaphore(2)
        self._rate_limiter = _TokenBucket(rate=8.0, capacity=8.0)
        self._pending: int = 0
        self._text_cache: Dict[str, str] = {}
        self._json_cache: Dict[str, dict] = {}
        # Negative cache for tickers with no SEC CIK — prevents bulk A-Z scans
        # from scanning the 10MB tickers map hundreds of times per second.
        self._cik_negative_cache: Dict[str, float] = {}
        self._cik_negative_ttl: float = 3600.0  # 1 hour
        self._cache_dir = "/tmp/stock_oracle_sec_cache"
        try:
            os.makedirs(self._cache_dir, exist_ok=True)
        except Exception:
            pass
        self._user_agent = f"{user_agent_name} ({settings.SEC_EMAIL})"
        self._session: Optional[aiohttp.ClientSession] = None

    async def _get_session(self) -> aiohttp.ClientSession:
        """Get or create a persistent session."""
        if self._session is None or self._session.closed:
            self._session = aiohttp.ClientSession(
                timeout=self.http_timeout,
                headers={"User-Agent": self._user_agent},
            )
        return self._session

    async def close(self) -> None:
        """Close the persistent session."""
        if self._session and not self._session.closed:
            await self._session.close()
            self._session = None

    # ------------------------------------------------------------------
    # Deadline management
    # ------------------------------------------------------------------

    def set_deadline(self, seconds_from_now: float) -> None:
        _deadline_var.set(_time.monotonic() + seconds_from_now)

    def clear_deadline(self) -> None:
        _deadline_var.set(None)

    @property
    def deadline(self) -> Optional[float]:
        return _deadline_var.get()

    @deadline.setter
    def deadline(self, value: Optional[float]) -> None:
        _deadline_var.set(value)

    def remaining_time(self) -> Optional[float]:
        dl = _deadline_var.get()
        if dl is None:
            return None
        return max(0.0, dl - _time.monotonic())

    def is_deadline_exceeded(self) -> bool:
        dl = _deadline_var.get()
        if dl is None:
            return False
        return _time.monotonic() >= dl

    # ------------------------------------------------------------------
    # Cache helpers
    # ------------------------------------------------------------------

    def _cache_path(self, url: str) -> str:
        h = hashlib.sha256(url.encode("utf-8")).hexdigest()
        return os.path.join(self._cache_dir, h)

    # ------------------------------------------------------------------
    # CIK lookup
    # ------------------------------------------------------------------

    async def get_company_cik(self, ticker: str) -> Optional[str]:
        """Look up zero-padded 10-digit CIK for a ticker."""
        tkr = ticker.upper()
        now = _time.monotonic()

        # Negative-cache hit: don't scan 10MB dict for known-missing tickers
        expiry = self._cik_negative_cache.get(tkr)
        if expiry is not None and now < expiry:
            return None

        url = f"{self.sec_base_www}/files/company_tickers.json"
        try:
            data = await self.fetch_json(url)
            for _key, company_info in data.items():
                if company_info.get("ticker", "").upper() == tkr:
                    cik_str = str(company_info.get("cik_str", "")).zfill(10)
                    logger.info(f"Found CIK {cik_str} for ticker {ticker}")
                    self._cik_negative_cache.pop(tkr, None)
                    return cik_str
            logger.warning(f"Ticker {ticker} not found in SEC mapping")
            self._cik_negative_cache[tkr] = now + self._cik_negative_ttl
            return None
        except Exception as e:
            logger.error(f"Error fetching CIK for {ticker}: {e}")
            return None

    # ------------------------------------------------------------------
    # HTTP fetch methods
    # ------------------------------------------------------------------

    async def fetch_json(self, url: str, skip_cache: bool = False) -> dict:
        """Fetch JSON with retry, backoff, and caching.

        Args:
            skip_cache: If True, bypass in-memory and disk cache reads (still writes
                        to cache after a successful fetch so subsequent calls benefit).
        """
        # In-memory cache
        if not skip_cache and url in self._json_cache:
            return self._json_cache[url]
        # Disk cache
        if not skip_cache:
            try:
                cp = self._cache_path(url) + ".json"
                if os.path.exists(cp):
                    ttl_sec = max(3600, settings.SEC_DATA_REFRESH_HOURS * 3600)
                    if _time.time() - os.path.getmtime(cp) <= ttl_sec:
                        with open(cp, "r", encoding="utf-8") as f:
                            data = json.load(f)
                            self._json_cache[url] = data
                            return data
            except Exception:
                pass

        # Backpressure: reject immediately if too many SEC requests are already pending.
        # This prevents thousands of coroutines from stacking up in the event loop,
        # which would starve health checks and other endpoints.
        if self._pending >= self._MAX_PENDING:
            raise RuntimeError(
                f"SEC request queue full ({self._pending}/{self._MAX_PENDING} pending)"
            )
        self._pending += 1
        try:
            attempts = 6
            backoff = 1.0
            last_exc = None
            for _i in range(attempts):
                now = _time.monotonic()
                dl = _deadline_var.get()
                if dl is not None and now >= dl:
                    break
                req_timeout = self.http_timeout
                if dl is not None:
                    remaining = max(0.0, dl - now)
                    if remaining < 0.25:
                        break
                    req_timeout = aiohttp.ClientTimeout(
                        total=min(remaining, getattr(self.http_timeout, "total", 12))
                    )
                async with self._req_sem:
                    await self._rate_limiter.acquire()
                    try:
                        session = await self._get_session()
                        async with session.get(url, timeout=req_timeout, headers={"Accept": "application/json"}) as resp:
                            if resp.status == 429:
                                retry_after = resp.headers.get("Retry-After")
                                delay = (
                                    float(retry_after)
                                    if retry_after and retry_after.isdigit()
                                    else backoff
                                )
                                dl = _deadline_var.get()
                                if dl is not None:
                                    remaining = max(0.0, dl - _time.monotonic())
                                    delay = min(delay, max(0.0, remaining - 0.05))
                                await asyncio.sleep(
                                    max(0.0, delay)
                                    + random.uniform(0.0, delay * 0.25 if delay > 0 else 0.0)
                                )
                                backoff *= 1.8
                                continue
                            if 500 <= resp.status < 600:
                                delay = backoff
                                dl = _deadline_var.get()
                                if dl is not None:
                                    remaining = max(0.0, dl - _time.monotonic())
                                    delay = min(delay, max(0.0, remaining - 0.05))
                                await asyncio.sleep(
                                    max(0.0, delay)
                                    + random.uniform(0.0, delay * 0.25 if delay > 0 else 0.0)
                                )
                                backoff *= 1.8
                                continue
                            resp.raise_for_status()
                            # SEC's filing index.json (used by Form 4 ingest) is
                            # served with text/html content-type, so disable the
                            # mimetype check.
                            data = await resp.json(content_type=None)
                            # Cache successful response
                            try:
                                with open(self._cache_path(url) + ".json", "w", encoding="utf-8") as f:
                                    json.dump(data, f)
                            except Exception:
                                pass
                            self._json_cache[url] = data
                            return data
                    except aiohttp.ClientResponseError as e:
                        if 400 <= e.status < 500:
                            raise  # 4xx: no retry, raise immediately
                        last_exc = e
                        delay = backoff
                        dl = _deadline_var.get()
                        if dl is not None:
                            remaining = max(0.0, dl - _time.monotonic())
                            delay = min(delay, max(0.0, remaining - 0.05))
                        await asyncio.sleep(
                            max(0.0, delay)
                            + random.uniform(0.0, delay * 0.25 if delay > 0 else 0.0)
                        )
                        backoff *= 1.8
                        continue
                    except Exception as e:
                        last_exc = e
                        delay = backoff
                        dl = _deadline_var.get()
                        if dl is not None:
                            remaining = max(0.0, dl - _time.monotonic())
                            delay = min(delay, max(0.0, remaining - 0.05))
                        await asyncio.sleep(
                            max(0.0, delay)
                            + random.uniform(0.0, delay * 0.25 if delay > 0 else 0.0)
                        )
                        backoff *= 1.8
                        continue
            raise last_exc if last_exc else RuntimeError("Failed to fetch JSON")
        finally:
            self._pending -= 1

    # ------------------------------------------------------------------
    # CIK reverse-lookup (numeric CIK → ticker)
    # ------------------------------------------------------------------

    async def get_ticker_for_cik(self, cik: str) -> Optional[str]:
        """Return the primary ticker for a numeric CIK, or None if not found."""
        cik_str = str(int(cik)).zfill(10)
        url = f"{self.sec_base_www}/files/company_tickers.json"
        try:
            data = await self.fetch_json(url)
            for _, info in data.items():
                if str(info.get("cik_str", "")).zfill(10) == cik_str:
                    return info.get("ticker", "").upper() or None
        except Exception as e:
            logger.error(f"Error in get_ticker_for_cik({cik}): {e}")
        return None

    # ------------------------------------------------------------------
    # Full-index / daily-index helpers
    # ------------------------------------------------------------------

    async def fetch_quarterly_company_idx(self, year: int, quarter: int) -> str:
        """Fetch the full-index company.idx for a given year/quarter (fixed-width text)."""
        url = f"{self.sec_base_www}/Archives/edgar/full-index/{year}/QTR{quarter}/company.idx"
        return await self.fetch_text(url, accept="text/plain")

    async def fetch_daily_index(self, date_str: str) -> str:
        """Fetch the daily company index for a given date string (YYYYMMDD).

        Short TTL: the disk cache key is unique per date, so yesterday's file is
        cached forever. Today's file may be growing — callers should use skip_cache
        if they need the freshest data.
        """
        from datetime import datetime
        dt = datetime.strptime(date_str, "%Y%m%d")
        quarter = (dt.month - 1) // 3 + 1
        url = f"{self.sec_base_www}/Archives/edgar/daily-index/{dt.year}/QTR{quarter}/company.{date_str}.idx"
        return await self.fetch_text(url, accept="text/plain")

    async def fetch_form345_zip(self, year: int, quarter: int, dest_path: str) -> str:
        """Stream the form345.zip full-index archive to dest_path. Returns dest_path."""
        import os
        url = f"{self.sec_base_www}/Archives/edgar/full-index/{year}/QTR{quarter}/form.idx"
        # form.idx is a fixed-width index file filtered to form types; for zip download
        # use the actual form345.zip:
        zip_url = f"{self.sec_base_www}/Archives/edgar/full-index/{year}/QTR{quarter}/form345.zip"
        if self._pending >= self._MAX_PENDING:
            raise RuntimeError(f"SEC request queue full ({self._pending}/{self._MAX_PENDING} pending)")
        self._pending += 1
        try:
            import aiohttp as _aiohttp
            session = await self._get_session()
            async with self._req_sem:
                await self._rate_limiter.acquire()
                async with session.get(zip_url, timeout=aiohttp.ClientTimeout(total=300)) as resp:
                    resp.raise_for_status()
                    os.makedirs(os.path.dirname(dest_path) or ".", exist_ok=True)
                    with open(dest_path, "wb") as f:
                        async for chunk in resp.content.iter_chunked(65536):
                            f.write(chunk)
            return dest_path
        finally:
            self._pending -= 1

    async def fetch_text(self, url: str, accept: str = "text/html", max_bytes: Optional[int] = None) -> str:
        """Fetch text content with retry, backoff, block page detection, and caching."""
        # In-memory cache
        if url in self._text_cache:
            return self._text_cache[url]
        # Disk cache
        try:
            cp = self._cache_path(url) + ".txt"
            if os.path.exists(cp):
                ttl_sec = max(3600, settings.SEC_DATA_REFRESH_HOURS * 3600)
                if _time.time() - os.path.getmtime(cp) <= ttl_sec:
                    with open(cp, "r", encoding="utf-8") as f:
                        text = f.read()
                        if _is_sec_block_page(text):
                            try:
                                os.remove(cp)
                            except Exception:
                                pass
                        else:
                            if len(text) <= _MAX_MEMORY_CACHE_BYTES:
                                self._text_cache[url] = text
                            return text
        except Exception:
            pass

        # Backpressure: reject immediately if too many SEC requests are already pending.
        if self._pending >= self._MAX_PENDING:
            raise RuntimeError(
                f"SEC request queue full ({self._pending}/{self._MAX_PENDING} pending)"
            )
        self._pending += 1
        try:
            attempts = 6
            backoff = 1.0
            last_exc = None
            for _i in range(attempts):
                now = _time.monotonic()
                dl = _deadline_var.get()
                if dl is not None and now >= dl:
                    break
                req_timeout = self.http_timeout
                if dl is not None:
                    remaining = max(0.0, dl - now)
                    if remaining < 0.25:
                        break
                    req_timeout = aiohttp.ClientTimeout(
                        total=min(remaining, getattr(self.http_timeout, "total", 12))
                    )
                async with self._req_sem:
                    await self._rate_limiter.acquire()
                    try:
                        session = await self._get_session()
                        async with session.get(url, timeout=req_timeout, headers={"Accept": accept}) as resp:
                            if resp.status == 429:
                                retry_after = resp.headers.get("Retry-After")
                                delay = (
                                    float(retry_after)
                                    if retry_after and retry_after.isdigit()
                                    else backoff
                                )
                                dl = _deadline_var.get()
                                if dl is not None:
                                    remaining = max(0.0, dl - _time.monotonic())
                                    delay = min(delay, max(0.0, remaining - 0.05))
                                await asyncio.sleep(
                                    max(0.0, delay)
                                    + random.uniform(0.0, delay * 0.25 if delay > 0 else 0.0)
                                )
                                backoff *= 1.8
                                continue
                            if 500 <= resp.status < 600:
                                delay = backoff
                                dl = _deadline_var.get()
                                if dl is not None:
                                    remaining = max(0.0, dl - _time.monotonic())
                                    delay = min(delay, max(0.0, remaining - 0.05))
                                await asyncio.sleep(
                                    max(0.0, delay)
                                    + random.uniform(0.0, delay * 0.25 if delay > 0 else 0.0)
                                )
                                backoff *= 1.8
                                continue
                            resp.raise_for_status()
                            if max_bytes is not None:
                                cl = resp.content_length
                                if cl is not None and cl > max_bytes:
                                    raise ValueError(
                                        f"Response too large ({cl} bytes > {max_bytes} limit)"
                                    )
                                raw = await resp.content.read(max_bytes + 1)
                                if len(raw) > max_bytes:
                                    raise ValueError(
                                        f"Response exceeded {max_bytes} byte limit"
                                    )
                                # Parse charset from Content-Type header directly;
                                # resp.get_encoding() needs resp.read() body which we didn't call.
                                ctype_hdr = resp.headers.get("Content-Type", "")
                                encoding = "utf-8"
                                for part in ctype_hdr.split(";"):
                                    part = part.strip()
                                    if part.lower().startswith("charset="):
                                        encoding = part[8:].strip().strip('"') or "utf-8"
                                        break
                                text = raw.decode(encoding, errors="replace")
                            else:
                                text = await resp.text()
                            if _is_sec_block_page(text):
                                last_exc = RuntimeError("SEC_BLOCKED")
                                delay = backoff * 2.0
                                dl = _deadline_var.get()
                                if dl is not None:
                                    remaining = max(0.0, dl - _time.monotonic())
                                    delay = min(delay, max(0.0, remaining - 0.05))
                                await asyncio.sleep(
                                    max(0.0, delay)
                                    + random.uniform(0.0, delay * 0.5 if delay > 0 else 0.0)
                                )
                                backoff *= 2.0
                                continue
                            # Cache successful response
                            if len(text) <= _MAX_MEMORY_CACHE_BYTES:
                                self._text_cache[url] = text
                            try:
                                with open(self._cache_path(url) + ".txt", "w", encoding="utf-8") as f:
                                    f.write(text)
                            except Exception:
                                pass
                            return text
                    except aiohttp.ClientResponseError as e:
                        if 400 <= e.status < 500:
                            raise  # 4xx: no retry, raise immediately
                        last_exc = e
                        delay = backoff
                        dl = _deadline_var.get()
                        if dl is not None:
                            remaining = max(0.0, dl - _time.monotonic())
                            delay = min(delay, max(0.0, remaining - 0.05))
                        await asyncio.sleep(
                            max(0.0, delay)
                            + random.uniform(0.0, delay * 0.25 if delay > 0 else 0.0)
                        )
                        backoff *= 1.8
                        continue
                    except ValueError:
                        raise  # Size limit exceeded: no retry (size won't change)
                    except Exception as e:
                        last_exc = e
                        delay = backoff
                        dl = _deadline_var.get()
                        if dl is not None:
                            remaining = max(0.0, dl - _time.monotonic())
                            delay = min(delay, max(0.0, remaining - 0.05))
                        await asyncio.sleep(
                            max(0.0, delay)
                            + random.uniform(0.0, delay * 0.25 if delay > 0 else 0.0)
                        )
                        backoff *= 1.8
                        continue
            raise last_exc if last_exc else RuntimeError("Failed to fetch text")
        finally:
            self._pending -= 1