"""Google Jobs adapter — scroll to load more results, JS extraction.""" from __future__ import annotations from loguru import logger from gimme_job.adapters.base import ManifestDrivenAdapter from gimme_job.adapters.registry import register from gimme_job.models.dto import RawJobCard _MAX_SCROLLS = 5 _SCROLL_WAIT_MS = 2000 _CARD_SELECTOR = 'div[jsname="y1Aese"]' _EXTRACT_JS = """() => { const cards = document.querySelectorAll('div[jsname="y1Aese"]'); return Array.from(cards).map(card => { const shareEl = card.closest('[data-share-url]'); const shareUrl = shareEl ? shareEl.getAttribute('data-share-url') : ''; // Build a clean udm=8 deep-link that opens the job detail panel. // The shareUrl has session-tied tokens (shmd, shmds, shem) that can expire; // we strip them and use only the htidocid + the new SPA fragment format. let jobUrl = ''; let htidocid = ''; if (shareUrl) { const docMatch = shareUrl.match(/htidocid=([^&]+)/); const qMatch = shareUrl.match(/[?&]q=([^&#]+)/); if (docMatch) { htidocid = decodeURIComponent(docMatch[1]); const q = qMatch ? qMatch[1] : 'orthodontist+job'; jobUrl = 'https://www.google.com/search?q=' + q + '&udm=8' + '#vhid=vt%3D20/docid%3D' + docMatch[1] + '&vssid=jobs-detail-viewer'; } } const dateSpan = card.querySelector('span[aria-label^="Posted"]'); const salarySpan = card.querySelector('span[aria-label^="Salary"]'); const typeSpan = card.querySelector('span[aria-label^="Employment Type"]'); // Positional extraction: filter aria-hidden and known UI texts, then take by index. // DOM order within a card is always: title → company → location. const UI_TEXTS = new Set(['Share', 'Click to copy link', 'Share link', 'Link copied']); const leafDivs = Array.from(card.querySelectorAll('div')).filter(d => d.children.length === 0 && d.getAttribute('aria-hidden') !== 'true' && d.textContent.trim() && !UI_TEXTS.has(d.textContent.trim()) ); const title = leafDivs[0]?.textContent.trim() || ''; const company = leafDivs[1]?.textContent.trim() || ''; // Location always contains " • via "; fallback to index 2 const locationEl = leafDivs.find(d => d.textContent.includes(' \u2022 via ')) || leafDivs[2]; const location = locationEl?.textContent.trim() || ''; return { title, company, location, url: jobUrl, external_job_id: htidocid, posted_text: dateSpan ? dateSpan.getAttribute('aria-label').replace('Posted ', '') : '', salary_text: salarySpan ? salarySpan.getAttribute('aria-label').replace('Salary ', '') : '', employment_type: typeSpan ? typeSpan.getAttribute('aria-label').replace('Employment Type ', '') : '', }; }); }""" @register("googlejobs") class GoogleJobsAdapter(ManifestDrivenAdapter): def collect_cards(self, page, manifest) -> list[RawJobCard]: """Scroll to load more results, then extract all cards via JS.""" self._scroll_to_load(page) return self._extract_cards(page) def paginate(self, page, page_index: int, config) -> bool: return False def _scroll_to_load(self, page) -> None: for i in range(_MAX_SCROLLS): prev_count = page.evaluate( f"document.querySelectorAll('{_CARD_SELECTOR}').length" ) page.evaluate("window.scrollTo(0, document.body.scrollHeight)") page.wait_for_timeout(_SCROLL_WAIT_MS) new_count = page.evaluate( f"document.querySelectorAll('{_CARD_SELECTOR}').length" ) logger.debug(f"[googlejobs] Scroll {i+1}: {prev_count} → {new_count} cards") if new_count <= prev_count: logger.debug(f"[googlejobs] No new cards after scroll {i+1} — stopping") break def _extract_cards(self, page) -> list[RawJobCard]: data = page.evaluate(_EXTRACT_JS) cards = [] for item in data: title = (item.get("title") or "").strip() if not title: continue cards.append(RawJobCard( title=title, company=(item.get("company") or "").strip() or None, location=(item.get("location") or "").strip() or None, url=item.get("url") or None, external_job_id=(item.get("external_job_id") or "").strip() or None, posted_text=(item.get("posted_text") or "").strip() or None, salary_text=(item.get("salary_text") or "").strip() or None, employment_type=(item.get("employment_type") or "").strip() or None, )) logger.debug(f"[googlejobs] Extracted {len(cards)} cards") return cards