"""Fingerprint-based deduplication against the database.""" from __future__ import annotations from sqlalchemy import select from sqlalchemy.orm import Session from gimme_job.models.db import JobPosting from gimme_job.models.dto import JobPostingCandidate from gimme_job.utils.hashing import compute_fingerprint def ensure_fingerprints(candidates: list[JobPostingCandidate]) -> list[JobPostingCandidate]: """Compute fingerprints for any candidates that don't have one.""" for c in candidates: if not c.fingerprint: c.fingerprint = compute_fingerprint( site_id=c.site_id, title=c.title, company=c.company, location=c.location, url=c.job_url, ) return candidates def deduplicate_in_batch(candidates: list[JobPostingCandidate]) -> list[JobPostingCandidate]: """Remove duplicates within the current batch by fingerprint.""" seen: set[str] = set() result = [] for c in candidates: if c.fingerprint and c.fingerprint not in seen: seen.add(c.fingerprint) result.append(c) return result