You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
53 lines
1.6 KiB
Python
53 lines
1.6 KiB
Python
import hashlib
|
|
import re
|
|
|
|
|
|
def normalize_for_fingerprint(text: str | None) -> str:
|
|
if not text:
|
|
return ""
|
|
# lowercase, strip, collapse whitespace, remove punctuation variation
|
|
t = text.lower().strip()
|
|
t = re.sub(r"\s+", " ", t)
|
|
return t
|
|
|
|
|
|
def compute_fingerprint(
|
|
site_id: str,
|
|
title: str | None,
|
|
company: str | None,
|
|
location: str | None,
|
|
url: str | None,
|
|
external_job_id: str | None = None,
|
|
) -> str:
|
|
"""Compute a SHA256 fingerprint for deduplication.
|
|
|
|
When external_job_id is provided (e.g. LinkedIn job ID, Google htidocid),
|
|
use site_id + external_job_id only — immune to title/location text drift.
|
|
Otherwise fall back to title+company+location+canonical_url.
|
|
"""
|
|
if external_job_id:
|
|
parts = [normalize_for_fingerprint(site_id), external_job_id.strip()]
|
|
else:
|
|
canonical_url = _canonical_url(url) if url else ""
|
|
parts = [
|
|
normalize_for_fingerprint(site_id),
|
|
normalize_for_fingerprint(title),
|
|
normalize_for_fingerprint(company),
|
|
normalize_for_fingerprint(location),
|
|
canonical_url,
|
|
]
|
|
raw = "|".join(parts)
|
|
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _canonical_url(url: str) -> str:
|
|
"""Strip tracking/session query parameters, keep path."""
|
|
from urllib.parse import urlparse, urlunparse
|
|
try:
|
|
parsed = urlparse(url)
|
|
# Drop query string entirely for canonicalization
|
|
canonical = urlunparse((parsed.scheme, parsed.netloc, parsed.path, "", "", ""))
|
|
return canonical.lower().rstrip("/")
|
|
except Exception:
|
|
return url.lower().strip()
|