gimme-job/gimme_job/adapters/googlejobs.py

"""Google Jobs adapter — scroll to load more results, JS extraction."""
from __future__ import annotations

from loguru import logger

from gimme_job.adapters.base import ManifestDrivenAdapter
from gimme_job.adapters.registry import register
from gimme_job.models.dto import RawJobCard

_MAX_SCROLLS = 5
_SCROLL_WAIT_MS = 2000
_CARD_SELECTOR = 'div[jsname="y1Aese"]'

_EXTRACT_JS = """() => {
    const cards = document.querySelectorAll('div[jsname="y1Aese"]');
    return Array.from(cards).map(card => {
        const shareEl = card.closest('[data-share-url]');
        const shareUrl = shareEl ? shareEl.getAttribute('data-share-url') : '';

        let htidocid = '';
        if (shareUrl) {
            const docMatch = shareUrl.match(/htidocid=([^&]+)/);
            if (docMatch) htidocid = decodeURIComponent(docMatch[1]);
        }

        // Use the first external apply URL (Indeed, Glassdoor, Monster, etc.)
        // pre-loaded inside the card's share_el. These are direct links to the
        // source posting and work regardless of session/location, unlike
        // Google Jobs htidocid deep-links which require the job to be in the
        // current search context.
        let jobUrl = '';
        if (shareEl) {
            const links = Array.from(shareEl.querySelectorAll('a[href^="https://"]'));
            const apply = links.find(a => {
                const href = a.getAttribute('href') || '';
                return !href.includes('://www.google.com')
                    && !href.includes('://google.com')
                    && !href.includes('gstatic.com')
                    && !href.includes('accounts.google.com');
            });
            if (apply) jobUrl = apply.getAttribute('href');
        }

        const dateSpan = card.querySelector('span[aria-label^="Posted"]');
        const salarySpan = card.querySelector('span[aria-label^="Salary"]');
        const typeSpan = card.querySelector('span[aria-label^="Employment Type"]');

        // Positional extraction: filter aria-hidden and known UI texts, then take by index.
        // DOM order within a card is always: title → company → location.
        const UI_TEXTS = new Set(['Share', 'Click to copy link', 'Share link', 'Link copied']);
        const leafDivs = Array.from(card.querySelectorAll('div')).filter(d =>
            d.children.length === 0 &&
            d.getAttribute('aria-hidden') !== 'true' &&
            d.textContent.trim() &&
            !UI_TEXTS.has(d.textContent.trim())
        );
        const title = leafDivs[0]?.textContent.trim() || '';
        const company = leafDivs[1]?.textContent.trim() || '';
        // Location always contains " • via "; fallback to index 2
        const locationEl = leafDivs.find(d => d.textContent.includes(' \u2022 via ')) || leafDivs[2];
        const location = locationEl?.textContent.trim() || '';

        return {
            title,
            company,
            location,
            url: jobUrl,
            external_job_id: htidocid,
            posted_text: dateSpan ? dateSpan.getAttribute('aria-label').replace('Posted ', '') : '',
            salary_text: salarySpan ? salarySpan.getAttribute('aria-label').replace('Salary ', '') : '',
            employment_type: typeSpan ? typeSpan.getAttribute('aria-label').replace('Employment Type ', '') : '',
        };
    });
}"""


@register("googlejobs")
class GoogleJobsAdapter(ManifestDrivenAdapter):

    def collect_cards(self, page, manifest) -> list[RawJobCard]:
        """Scroll to load more results, then extract all cards via JS."""
        self._scroll_to_load(page)
        return self._extract_cards(page)

    def paginate(self, page, page_index: int, config) -> bool:
        return False

    def _scroll_to_load(self, page) -> None:
        for i in range(_MAX_SCROLLS):
            prev_count = page.evaluate(
                f"document.querySelectorAll('{_CARD_SELECTOR}').length"
            )
            page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
            page.wait_for_timeout(_SCROLL_WAIT_MS)
            new_count = page.evaluate(
                f"document.querySelectorAll('{_CARD_SELECTOR}').length"
            )
            logger.debug(f"[googlejobs] Scroll {i+1}: {prev_count} → {new_count} cards")
            if new_count <= prev_count:
                logger.debug(f"[googlejobs] No new cards after scroll {i+1} — stopping")
                break

    def _extract_cards(self, page) -> list[RawJobCard]:
        data = page.evaluate(_EXTRACT_JS)
        cards = []
        for item in data:
            title = (item.get("title") or "").strip()
            if not title:
                continue
            cards.append(RawJobCard(
                title=title,
                company=(item.get("company") or "").strip() or None,
                location=(item.get("location") or "").strip() or None,
                url=item.get("url") or None,
                external_job_id=(item.get("external_job_id") or "").strip() or None,
                posted_text=(item.get("posted_text") or "").strip() or None,
                salary_text=(item.get("salary_text") or "").strip() or None,
                employment_type=(item.get("employment_type") or "").strip() or None,
            ))
        logger.debug(f"[googlejobs] Extracted {len(cards)} cards")
        return cards