You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
61 lines
1.8 KiB
Python
61 lines
1.8 KiB
Python
"""Tests for deduplication logic."""
|
|
from gimme_job.models.dto import JobPostingCandidate
|
|
from gimme_job.runtime.dedupe import deduplicate_in_batch, ensure_fingerprints
|
|
from gimme_job.utils.hashing import compute_fingerprint
|
|
|
|
|
|
def _make_candidate(title: str, company: str = "ABC", url: str = None) -> JobPostingCandidate:
|
|
fp = compute_fingerprint("indeed", title, company, "AZ", url)
|
|
return JobPostingCandidate(
|
|
site_id="indeed",
|
|
title=title,
|
|
company=company,
|
|
location="AZ",
|
|
job_url=url,
|
|
fingerprint=fp,
|
|
)
|
|
|
|
|
|
def test_deduplicate_removes_duplicates():
|
|
c1 = _make_candidate("Orthodontist", url="https://example.com/1")
|
|
c2 = _make_candidate("Orthodontist", url="https://example.com/1") # same fingerprint
|
|
c3 = _make_candidate("Dentist", url="https://example.com/2")
|
|
|
|
result = deduplicate_in_batch([c1, c2, c3])
|
|
assert len(result) == 2
|
|
titles = {r.title for r in result}
|
|
assert "Orthodontist" in titles
|
|
assert "Dentist" in titles
|
|
|
|
|
|
def test_deduplicate_empty_list():
|
|
assert deduplicate_in_batch([]) == []
|
|
|
|
|
|
def test_ensure_fingerprints():
|
|
c = JobPostingCandidate(
|
|
site_id="indeed",
|
|
title="Orthodontist",
|
|
company="ABC",
|
|
location="AZ",
|
|
job_url="https://example.com/1",
|
|
fingerprint="", # empty
|
|
)
|
|
result = ensure_fingerprints([c])
|
|
assert result[0].fingerprint != ""
|
|
assert len(result[0].fingerprint) == 64
|
|
|
|
|
|
def test_fingerprint_not_overwritten():
|
|
fp = compute_fingerprint("indeed", "Orthodontist", "ABC", "AZ", "https://example.com/1")
|
|
c = JobPostingCandidate(
|
|
site_id="indeed",
|
|
title="Orthodontist",
|
|
company="ABC",
|
|
location="AZ",
|
|
job_url="https://example.com/1",
|
|
fingerprint=fp,
|
|
)
|
|
result = ensure_fingerprints([c])
|
|
assert result[0].fingerprint == fp
|