LinkedIn: fix relative URLs and deduplicate accessibility text in titles

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
main
I Luk Kim 5 months ago
parent c4f9d5638b
commit 317f929059

@ -8,6 +8,30 @@ from gimme_job.adapters.registry import register
from gimme_job.models.dto import JobPostingCandidate, RawJobCard from gimme_job.models.dto import JobPostingCandidate, RawJobCard
def _deduplicate_title(title: str) -> str:
"""Remove LinkedIn's accessibility text duplication.
LinkedIn renders job titles as:
<span aria-hidden>Title</span>
<span class="accessible">Title with verification</span>
inner_text() returns both, so we get "Title Title with verification".
Strategy: if the title starts with a repeated prefix word-by-word, keep the shorter part.
"""
words = title.split()
n = len(words)
# Try splitting at each midpoint from 1 to n//2
for split in range(1, n // 2 + 1):
prefix = words[:split]
rest = words[split:]
# Exact full duplication: "Foo Bar Foo Bar"
if rest == prefix:
return " ".join(prefix)
# Prefix repeats at start of rest: "Foo Foo with verification"
if rest[:split] == prefix:
return " ".join(prefix)
return title
@register("linkedin") @register("linkedin")
class LinkedInAdapter(ManifestDrivenAdapter): class LinkedInAdapter(ManifestDrivenAdapter):
@ -17,6 +41,16 @@ class LinkedInAdapter(ManifestDrivenAdapter):
return super().collect_cards(page, manifest) return super().collect_cards(page, manifest)
def normalize(self, raw: RawJobCard) -> JobPostingCandidate: def normalize(self, raw: RawJobCard) -> JobPostingCandidate:
# Fix relative URLs → absolute
if raw.url and raw.url.startswith("/"):
raw = raw.model_copy(update={"url": "https://www.linkedin.com" + raw.url})
# Fix duplicated titles caused by LinkedIn's accessibility span
# e.g. "Orthodontist Orthodontist with verification" → "Orthodontist"
# e.g. "LOCUM Dentist ... LOCUM Dentist ..." → "LOCUM Dentist ..."
if raw.title:
raw = raw.model_copy(update={"title": _deduplicate_title(raw.title)})
return super().normalize(raw) return super().normalize(raw)
@staticmethod @staticmethod

Loading…
Cancel
Save