import hashlib import re def normalize_for_fingerprint(text: str | None) -> str: if not text: return "" # lowercase, strip, collapse whitespace, remove punctuation variation t = text.lower().strip() t = re.sub(r"\s+", " ", t) return t def compute_fingerprint( site_id: str, title: str | None, company: str | None, location: str | None, url: str | None, ) -> str: """Compute a SHA256 fingerprint for deduplication. URL is canonical-ized (query params stripped) if present. Falls back to title+company+location when URL is absent. """ canonical_url = _canonical_url(url) if url else "" parts = [ normalize_for_fingerprint(site_id), normalize_for_fingerprint(title), normalize_for_fingerprint(company), normalize_for_fingerprint(location), canonical_url, ] raw = "|".join(parts) return hashlib.sha256(raw.encode("utf-8")).hexdigest() def _canonical_url(url: str) -> str: """Strip tracking/session query parameters, keep path.""" from urllib.parse import urlparse, urlunparse try: parsed = urlparse(url) # Drop query string entirely for canonicalization canonical = urlunparse((parsed.scheme, parsed.netloc, parsed.path, "", "", "")) return canonical.lower().rstrip("/") except Exception: return url.lower().strip()