You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
35 lines
1.1 KiB
Python
35 lines
1.1 KiB
Python
"""Fingerprint-based deduplication against the database."""
|
|
from __future__ import annotations
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import Session
|
|
|
|
from gimme_job.models.db import JobPosting
|
|
from gimme_job.models.dto import JobPostingCandidate
|
|
from gimme_job.utils.hashing import compute_fingerprint
|
|
|
|
|
|
def ensure_fingerprints(candidates: list[JobPostingCandidate]) -> list[JobPostingCandidate]:
|
|
"""Compute fingerprints for any candidates that don't have one."""
|
|
for c in candidates:
|
|
if not c.fingerprint:
|
|
c.fingerprint = compute_fingerprint(
|
|
site_id=c.site_id,
|
|
title=c.title,
|
|
company=c.company,
|
|
location=c.location,
|
|
url=c.job_url,
|
|
)
|
|
return candidates
|
|
|
|
|
|
def deduplicate_in_batch(candidates: list[JobPostingCandidate]) -> list[JobPostingCandidate]:
|
|
"""Remove duplicates within the current batch by fingerprint."""
|
|
seen: set[str] = set()
|
|
result = []
|
|
for c in candidates:
|
|
if c.fingerprint and c.fingerprint not in seen:
|
|
seen.add(c.fingerprint)
|
|
result.append(c)
|
|
return result
|