You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
58 lines
1.7 KiB
Python
58 lines
1.7 KiB
Python
"""Unit tests for text normalizer."""
|
|
|
|
|
|
def test_html_to_text():
|
|
from libs.parser.text_normalizer import html_to_text
|
|
html = "<p>Hello <b>world</b></p>"
|
|
text = html_to_text(html)
|
|
assert "Hello" in text
|
|
assert "world" in text
|
|
assert "<p>" not in text
|
|
|
|
|
|
def test_normalize_unicode_quotes():
|
|
from libs.parser.text_normalizer import normalize_unicode
|
|
text = "\u2018Hello\u2019 \u201cworld\u201d"
|
|
result = normalize_unicode(text)
|
|
assert "'" in result
|
|
assert '"' in result
|
|
|
|
|
|
def test_normalize_em_dash():
|
|
from libs.parser.text_normalizer import normalize_unicode
|
|
text = "Q4\u2014best quarter"
|
|
result = normalize_unicode(text)
|
|
assert " - " in result
|
|
|
|
|
|
def test_collapse_whitespace():
|
|
from libs.parser.text_normalizer import collapse_whitespace
|
|
text = "Hello\n\n\n\n\nWorld"
|
|
result = collapse_whitespace(text)
|
|
assert "Hello" in result
|
|
assert "World" in result
|
|
|
|
|
|
def test_normalize_text_pipeline():
|
|
from libs.parser.text_normalizer import normalize_text
|
|
html = "<html><body><p>Revenue guidance raised. Demand remains strong.</p></body></html>"
|
|
result = normalize_text(html, is_html=True)
|
|
assert "Revenue" in result
|
|
assert "<p>" not in result
|
|
|
|
|
|
def test_script_removed():
|
|
from libs.parser.text_normalizer import normalize_text
|
|
html = "<html><body><script>var x=1;</script><p>Content</p></body></html>"
|
|
result = normalize_text(html, is_html=True)
|
|
assert "var x" not in result
|
|
assert "Content" in result
|
|
|
|
|
|
def test_looks_like_html_detects_sec_exhibit_markup():
|
|
from libs.parser.text_normalizer import looks_like_html
|
|
|
|
html = "<DOCUMENT><TEXT><html><body><div>Content</div></body></html>"
|
|
|
|
assert looks_like_html(html) is True
|