From 72bcffad7f80c2339101e8b9d95dff60cc2f9325 Mon Sep 17 00:00:00 2001 From: I Luk Kim Date: Sat, 28 Mar 2026 00:18:15 -0700 Subject: [PATCH] Add HRSA Health Workforce Connector adapter Form-based keyword search (PrimeNG Angular SPA) with client-side pagination. Uses JS click to bypass headless visibility issue on the Search button. Extracts 112 dentist opportunity cards per run. Co-Authored-By: Claude Sonnet 4.6 --- gimme_job/adapters/hrsa.py | 131 +++++++++++++++++++++++++++++++++++++ sites/hrsa.yaml | 50 ++++++++++++++ 2 files changed, 181 insertions(+) create mode 100644 gimme_job/adapters/hrsa.py create mode 100644 sites/hrsa.yaml diff --git a/gimme_job/adapters/hrsa.py b/gimme_job/adapters/hrsa.py new file mode 100644 index 0000000..10192d6 --- /dev/null +++ b/gimme_job/adapters/hrsa.py @@ -0,0 +1,131 @@ +"""HRSA Health Workforce Connector adapter — form keyword search, client-side pagination.""" +from __future__ import annotations + +from loguru import logger + +from gimme_job.adapters.base import ManifestDrivenAdapter +from gimme_job.adapters.registry import register +from gimme_job.models.dto import JobPostingCandidate, RawJobCard + +_DEFAULT_KEYWORD = "dentist" + + +@register("hrsa") +class HRSAConnectorAdapter(ManifestDrivenAdapter): + + def apply_search(self, page, query) -> None: + """Type keyword into the search form and submit.""" + keyword = (query.keywords[0] if query.keywords else _DEFAULT_KEYWORD) + + # Wait for initial JS render to complete + try: + page.wait_for_load_state("networkidle", timeout=20000) + except Exception: + pass + + # Dismiss feedback dialog if present + try: + no_thanks = page.locator('button:has-text("No Thanks")').first + if no_thanks.is_visible(): + no_thanks.click() + page.wait_for_timeout(500) + except Exception: + pass + + # Wait for the main keyword input (not the banner header input) + kw_locator = page.locator('[placeholder="Keyword, Name, or Job Title"]').first + try: + kw_locator.wait_for(state="visible", timeout=15000) + except Exception: + logger.warning("[hrsa] Keyword input not visible — skipping search") + return + + kw_locator.fill(keyword) + + # Press Enter or JS-click the Search button (bypasses Playwright visibility check) + try: + page.evaluate("""() => { + const btn = document.querySelector('button[label="Search"]') || + Array.from(document.querySelectorAll('button')) + .find(b => b.textContent.trim() === 'Search'); + if (btn) btn.click(); + }""") + except Exception: + kw_locator.press("Enter") + + # Wait for results to load + try: + page.wait_for_load_state("networkidle", timeout=15000) + except Exception: + pass + + logger.debug(f"[hrsa] Search submitted for keyword: '{keyword}'") + + def collect_cards(self, page, manifest) -> list[RawJobCard]: + """Extract job opportunity rows from the results table.""" + raw_data = page.evaluate("""() => { + const links = document.querySelectorAll( + 'a[href*="/site-profile/"][href*="/opportunity/"]' + ); + return Array.from(links).map(a => { + const td = a.closest('td'); + if (!td) return null; + const tr = td.closest('tr'); + const cells = tr ? Array.from(tr.querySelectorAll('td')) : []; + const cell1 = cells[0]; + const cell2 = cells[1]; + + const c1kids = cell1 ? Array.from(cell1.children) : []; + // c1kids[0] = list (title link + specialty) + // c1kids[1] = "Job" + // c1kids[2] = employment type (Full-Time / Part-Time) + // c1kids[3] = site/company name + + const employmentType = c1kids[2]?.innerText?.trim() || null; + const company = c1kids[3]?.innerText?.trim() || null; + + const c2kids = cell2 ? Array.from(cell2.children) : []; + // c2kids[0] = programs + status + // c2kids[1] = location + const location = c2kids[1]?.innerText?.trim() || null; + + return { + title: a.innerText.trim(), + url: a.href, + company, + location, + employment_type: employmentType, + }; + }).filter(j => j && j.title && j.url.includes('/opportunity/')); + }""") + + cards = [] + for item in raw_data: + cards.append(RawJobCard( + title=item["title"], + company=item["company"], + location=item["location"], + employment_type=item["employment_type"], + url=item["url"], + )) + + logger.info(f"[hrsa] Extracted {len(cards)} cards") + return cards + + def paginate(self, page, page_index: int, config) -> bool: + """Click 'Go to next page' button if available (client-side SPA pagination).""" + next_btn = page.query_selector('button[aria-label="Go to next page"]') + if not next_btn or not next_btn.is_enabled(): + return False + + next_btn.click() + try: + page.wait_for_load_state("networkidle", timeout=15000) + except Exception: + pass + + logger.debug(f"[hrsa] Navigated to page {page_index + 2}") + return True + + def normalize(self, raw: RawJobCard) -> JobPostingCandidate: + return super().normalize(raw) diff --git a/sites/hrsa.yaml b/sites/hrsa.yaml new file mode 100644 index 0000000..d8852d2 --- /dev/null +++ b/sites/hrsa.yaml @@ -0,0 +1,50 @@ +site_id: hrsa +enabled: true +repair_needed: false + +identity: + label: HRSA Health Workforce Connector + category: government + base_url: https://connector.hrsa.gov + start_url_template: "https://connector.hrsa.gov/connector/search" + +browser: + profile_mode: persistent_chrome_profile + profile_name: JobAgent + headed_on_learn: true + headless_on_run: true + +search: + keyword_mode: none + location_mode: none + sort_mode: none + date_mode: none + result_list_wait_selector: "a[href*='/site-profile/'][href*='/opportunity/']" + +search_override: + keywords: + - dentist + +pagination: + mode: none + max_pages: 10 + +extract: + container_selectors: + - "a[href*='/site-profile/'][href*='/opportunity/']" + fields: + title: + text: [] + +post_filters: + include_posted_text: [] + +healthcheck: + min_items_expected: 0 + required_fields: + - title + fail_if_zero_when_known_active: false + +learn: + last_learned_at: null + source_url: null