You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
58 lines
1.8 KiB
Python
58 lines
1.8 KiB
Python
"""HTML-to-text conversion and text normalization."""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import unicodedata
|
|
|
|
from bs4 import BeautifulSoup
|
|
|
|
_BOILERPLATE_PATTERNS = [
|
|
re.compile(r"safe harbor.*?forward.looking statement", re.IGNORECASE | re.DOTALL),
|
|
re.compile(r"this press release.*?private securities litigation", re.IGNORECASE | re.DOTALL),
|
|
re.compile(r"^\s*page \d+ of \d+\s*$", re.IGNORECASE | re.MULTILINE),
|
|
re.compile(r"^\s*\[?\s*table of contents\s*\]?\s*$", re.IGNORECASE | re.MULTILINE),
|
|
]
|
|
|
|
_WHITESPACE = re.compile(r"\s{3,}")
|
|
|
|
|
|
def html_to_text(html: str) -> str:
|
|
"""Convert HTML to plain text using BeautifulSoup."""
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
# Remove script/style
|
|
for tag in soup(["script", "style", "head"]):
|
|
tag.decompose()
|
|
return soup.get_text(separator="\n")
|
|
|
|
|
|
def normalize_unicode(text: str) -> str:
|
|
"""Normalize unicode to NFC and replace fancy quotes/dashes."""
|
|
text = unicodedata.normalize("NFC", text)
|
|
# Fancy quotes → standard
|
|
text = text.replace("\u2018", "'").replace("\u2019", "'")
|
|
text = text.replace("\u201c", '"').replace("\u201d", '"')
|
|
# Em/en dash → hyphen
|
|
text = text.replace("\u2014", " - ").replace("\u2013", " - ")
|
|
return text
|
|
|
|
|
|
def remove_boilerplate(text: str) -> str:
|
|
"""Strip common boilerplate sections."""
|
|
for pattern in _BOILERPLATE_PATTERNS:
|
|
text = pattern.sub(" ", text)
|
|
return text
|
|
|
|
|
|
def collapse_whitespace(text: str) -> str:
|
|
"""Collapse runs of 3+ whitespace chars to double newline."""
|
|
return _WHITESPACE.sub("\n\n", text).strip()
|
|
|
|
|
|
def normalize_text(raw: str, is_html: bool = False) -> str:
|
|
"""Full normalization pipeline."""
|
|
text = html_to_text(raw) if is_html else raw
|
|
text = normalize_unicode(text)
|
|
text = remove_boilerplate(text)
|
|
text = collapse_whitespace(text)
|
|
return text
|