Add HRSA Health Workforce Connector adapter
Form-based keyword search (PrimeNG Angular SPA) with client-side pagination. Uses JS click to bypass headless visibility issue on the Search button. Extracts 112 dentist opportunity cards per run. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>main
parent
9d5251cead
commit
72bcffad7f
@ -0,0 +1,131 @@
|
|||||||
|
"""HRSA Health Workforce Connector adapter — form keyword search, client-side pagination."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
from gimme_job.adapters.base import ManifestDrivenAdapter
|
||||||
|
from gimme_job.adapters.registry import register
|
||||||
|
from gimme_job.models.dto import JobPostingCandidate, RawJobCard
|
||||||
|
|
||||||
|
_DEFAULT_KEYWORD = "dentist"
|
||||||
|
|
||||||
|
|
||||||
|
@register("hrsa")
|
||||||
|
class HRSAConnectorAdapter(ManifestDrivenAdapter):
|
||||||
|
|
||||||
|
def apply_search(self, page, query) -> None:
|
||||||
|
"""Type keyword into the search form and submit."""
|
||||||
|
keyword = (query.keywords[0] if query.keywords else _DEFAULT_KEYWORD)
|
||||||
|
|
||||||
|
# Wait for initial JS render to complete
|
||||||
|
try:
|
||||||
|
page.wait_for_load_state("networkidle", timeout=20000)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Dismiss feedback dialog if present
|
||||||
|
try:
|
||||||
|
no_thanks = page.locator('button:has-text("No Thanks")').first
|
||||||
|
if no_thanks.is_visible():
|
||||||
|
no_thanks.click()
|
||||||
|
page.wait_for_timeout(500)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Wait for the main keyword input (not the banner header input)
|
||||||
|
kw_locator = page.locator('[placeholder="Keyword, Name, or Job Title"]').first
|
||||||
|
try:
|
||||||
|
kw_locator.wait_for(state="visible", timeout=15000)
|
||||||
|
except Exception:
|
||||||
|
logger.warning("[hrsa] Keyword input not visible — skipping search")
|
||||||
|
return
|
||||||
|
|
||||||
|
kw_locator.fill(keyword)
|
||||||
|
|
||||||
|
# Press Enter or JS-click the Search button (bypasses Playwright visibility check)
|
||||||
|
try:
|
||||||
|
page.evaluate("""() => {
|
||||||
|
const btn = document.querySelector('button[label="Search"]') ||
|
||||||
|
Array.from(document.querySelectorAll('button'))
|
||||||
|
.find(b => b.textContent.trim() === 'Search');
|
||||||
|
if (btn) btn.click();
|
||||||
|
}""")
|
||||||
|
except Exception:
|
||||||
|
kw_locator.press("Enter")
|
||||||
|
|
||||||
|
# Wait for results to load
|
||||||
|
try:
|
||||||
|
page.wait_for_load_state("networkidle", timeout=15000)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
logger.debug(f"[hrsa] Search submitted for keyword: '{keyword}'")
|
||||||
|
|
||||||
|
def collect_cards(self, page, manifest) -> list[RawJobCard]:
|
||||||
|
"""Extract job opportunity rows from the results table."""
|
||||||
|
raw_data = page.evaluate("""() => {
|
||||||
|
const links = document.querySelectorAll(
|
||||||
|
'a[href*="/site-profile/"][href*="/opportunity/"]'
|
||||||
|
);
|
||||||
|
return Array.from(links).map(a => {
|
||||||
|
const td = a.closest('td');
|
||||||
|
if (!td) return null;
|
||||||
|
const tr = td.closest('tr');
|
||||||
|
const cells = tr ? Array.from(tr.querySelectorAll('td')) : [];
|
||||||
|
const cell1 = cells[0];
|
||||||
|
const cell2 = cells[1];
|
||||||
|
|
||||||
|
const c1kids = cell1 ? Array.from(cell1.children) : [];
|
||||||
|
// c1kids[0] = list (title link + specialty)
|
||||||
|
// c1kids[1] = "Job"
|
||||||
|
// c1kids[2] = employment type (Full-Time / Part-Time)
|
||||||
|
// c1kids[3] = site/company name
|
||||||
|
|
||||||
|
const employmentType = c1kids[2]?.innerText?.trim() || null;
|
||||||
|
const company = c1kids[3]?.innerText?.trim() || null;
|
||||||
|
|
||||||
|
const c2kids = cell2 ? Array.from(cell2.children) : [];
|
||||||
|
// c2kids[0] = programs + status
|
||||||
|
// c2kids[1] = location
|
||||||
|
const location = c2kids[1]?.innerText?.trim() || null;
|
||||||
|
|
||||||
|
return {
|
||||||
|
title: a.innerText.trim(),
|
||||||
|
url: a.href,
|
||||||
|
company,
|
||||||
|
location,
|
||||||
|
employment_type: employmentType,
|
||||||
|
};
|
||||||
|
}).filter(j => j && j.title && j.url.includes('/opportunity/'));
|
||||||
|
}""")
|
||||||
|
|
||||||
|
cards = []
|
||||||
|
for item in raw_data:
|
||||||
|
cards.append(RawJobCard(
|
||||||
|
title=item["title"],
|
||||||
|
company=item["company"],
|
||||||
|
location=item["location"],
|
||||||
|
employment_type=item["employment_type"],
|
||||||
|
url=item["url"],
|
||||||
|
))
|
||||||
|
|
||||||
|
logger.info(f"[hrsa] Extracted {len(cards)} cards")
|
||||||
|
return cards
|
||||||
|
|
||||||
|
def paginate(self, page, page_index: int, config) -> bool:
|
||||||
|
"""Click 'Go to next page' button if available (client-side SPA pagination)."""
|
||||||
|
next_btn = page.query_selector('button[aria-label="Go to next page"]')
|
||||||
|
if not next_btn or not next_btn.is_enabled():
|
||||||
|
return False
|
||||||
|
|
||||||
|
next_btn.click()
|
||||||
|
try:
|
||||||
|
page.wait_for_load_state("networkidle", timeout=15000)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
logger.debug(f"[hrsa] Navigated to page {page_index + 2}")
|
||||||
|
return True
|
||||||
|
|
||||||
|
def normalize(self, raw: RawJobCard) -> JobPostingCandidate:
|
||||||
|
return super().normalize(raw)
|
||||||
@ -0,0 +1,50 @@
|
|||||||
|
site_id: hrsa
|
||||||
|
enabled: true
|
||||||
|
repair_needed: false
|
||||||
|
|
||||||
|
identity:
|
||||||
|
label: HRSA Health Workforce Connector
|
||||||
|
category: government
|
||||||
|
base_url: https://connector.hrsa.gov
|
||||||
|
start_url_template: "https://connector.hrsa.gov/connector/search"
|
||||||
|
|
||||||
|
browser:
|
||||||
|
profile_mode: persistent_chrome_profile
|
||||||
|
profile_name: JobAgent
|
||||||
|
headed_on_learn: true
|
||||||
|
headless_on_run: true
|
||||||
|
|
||||||
|
search:
|
||||||
|
keyword_mode: none
|
||||||
|
location_mode: none
|
||||||
|
sort_mode: none
|
||||||
|
date_mode: none
|
||||||
|
result_list_wait_selector: "a[href*='/site-profile/'][href*='/opportunity/']"
|
||||||
|
|
||||||
|
search_override:
|
||||||
|
keywords:
|
||||||
|
- dentist
|
||||||
|
|
||||||
|
pagination:
|
||||||
|
mode: none
|
||||||
|
max_pages: 10
|
||||||
|
|
||||||
|
extract:
|
||||||
|
container_selectors:
|
||||||
|
- "a[href*='/site-profile/'][href*='/opportunity/']"
|
||||||
|
fields:
|
||||||
|
title:
|
||||||
|
text: []
|
||||||
|
|
||||||
|
post_filters:
|
||||||
|
include_posted_text: []
|
||||||
|
|
||||||
|
healthcheck:
|
||||||
|
min_items_expected: 0
|
||||||
|
required_fields:
|
||||||
|
- title
|
||||||
|
fail_if_zero_when_known_active: false
|
||||||
|
|
||||||
|
learn:
|
||||||
|
last_learned_at: null
|
||||||
|
source_url: null
|
||||||
Loading…
Reference in New Issue