"""HTML-to-text conversion and text normalization."""
from __future__ import annotations

import re
import unicodedata

from bs4 import BeautifulSoup

_BOILERPLATE_PATTERNS = [
    re.compile(r"safe harbor.*?forward.looking statement", re.IGNORECASE | re.DOTALL),
    re.compile(r"this press release.*?private securities litigation", re.IGNORECASE | re.DOTALL),
    re.compile(r"^\s*page \d+ of \d+\s*$", re.IGNORECASE | re.MULTILINE),
    re.compile(r"^\s*\[?\s*table of contents\s*\]?\s*$", re.IGNORECASE | re.MULTILINE),
]

_WHITESPACE = re.compile(r"\s{3,}")


def html_to_text(html: str) -> str:
    """Convert HTML to plain text using BeautifulSoup."""
    soup = BeautifulSoup(html, "html.parser")
    # Remove script/style
    for tag in soup(["script", "style", "head"]):
        tag.decompose()
    return soup.get_text(separator="\n")


def normalize_unicode(text: str) -> str:
    """Normalize unicode to NFC and replace fancy quotes/dashes."""
    text = unicodedata.normalize("NFC", text)
    # Fancy quotes → standard
    text = text.replace("\u2018", "'").replace("\u2019", "'")
    text = text.replace("\u201c", '"').replace("\u201d", '"')
    # Em/en dash → hyphen
    text = text.replace("\u2014", " - ").replace("\u2013", " - ")
    return text


def remove_boilerplate(text: str) -> str:
    """Strip common boilerplate sections."""
    for pattern in _BOILERPLATE_PATTERNS:
        text = pattern.sub(" ", text)
    return text


def collapse_whitespace(text: str) -> str:
    """Collapse runs of 3+ whitespace chars to double newline."""
    return _WHITESPACE.sub("\n\n", text).strip()


def normalize_text(raw: str, is_html: bool = False) -> str:
    """Full normalization pipeline."""
    text = html_to_text(raw) if is_html else raw
    text = normalize_unicode(text)
    text = remove_boilerplate(text)
    text = collapse_whitespace(text)
    return text