Add HRSA Health Workforce Connector adapter
Form-based keyword search (PrimeNG Angular SPA) with client-side pagination. Uses JS click to bypass headless visibility issue on the Search button. Extracts 112 dentist opportunity cards per run. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>main
parent
9d5251cead
commit
72bcffad7f
@ -0,0 +1,131 @@
|
||||
"""HRSA Health Workforce Connector adapter — form keyword search, client-side pagination."""
|
||||
from __future__ import annotations
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from gimme_job.adapters.base import ManifestDrivenAdapter
|
||||
from gimme_job.adapters.registry import register
|
||||
from gimme_job.models.dto import JobPostingCandidate, RawJobCard
|
||||
|
||||
_DEFAULT_KEYWORD = "dentist"
|
||||
|
||||
|
||||
@register("hrsa")
|
||||
class HRSAConnectorAdapter(ManifestDrivenAdapter):
|
||||
|
||||
def apply_search(self, page, query) -> None:
|
||||
"""Type keyword into the search form and submit."""
|
||||
keyword = (query.keywords[0] if query.keywords else _DEFAULT_KEYWORD)
|
||||
|
||||
# Wait for initial JS render to complete
|
||||
try:
|
||||
page.wait_for_load_state("networkidle", timeout=20000)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Dismiss feedback dialog if present
|
||||
try:
|
||||
no_thanks = page.locator('button:has-text("No Thanks")').first
|
||||
if no_thanks.is_visible():
|
||||
no_thanks.click()
|
||||
page.wait_for_timeout(500)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Wait for the main keyword input (not the banner header input)
|
||||
kw_locator = page.locator('[placeholder="Keyword, Name, or Job Title"]').first
|
||||
try:
|
||||
kw_locator.wait_for(state="visible", timeout=15000)
|
||||
except Exception:
|
||||
logger.warning("[hrsa] Keyword input not visible — skipping search")
|
||||
return
|
||||
|
||||
kw_locator.fill(keyword)
|
||||
|
||||
# Press Enter or JS-click the Search button (bypasses Playwright visibility check)
|
||||
try:
|
||||
page.evaluate("""() => {
|
||||
const btn = document.querySelector('button[label="Search"]') ||
|
||||
Array.from(document.querySelectorAll('button'))
|
||||
.find(b => b.textContent.trim() === 'Search');
|
||||
if (btn) btn.click();
|
||||
}""")
|
||||
except Exception:
|
||||
kw_locator.press("Enter")
|
||||
|
||||
# Wait for results to load
|
||||
try:
|
||||
page.wait_for_load_state("networkidle", timeout=15000)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
logger.debug(f"[hrsa] Search submitted for keyword: '{keyword}'")
|
||||
|
||||
def collect_cards(self, page, manifest) -> list[RawJobCard]:
|
||||
"""Extract job opportunity rows from the results table."""
|
||||
raw_data = page.evaluate("""() => {
|
||||
const links = document.querySelectorAll(
|
||||
'a[href*="/site-profile/"][href*="/opportunity/"]'
|
||||
);
|
||||
return Array.from(links).map(a => {
|
||||
const td = a.closest('td');
|
||||
if (!td) return null;
|
||||
const tr = td.closest('tr');
|
||||
const cells = tr ? Array.from(tr.querySelectorAll('td')) : [];
|
||||
const cell1 = cells[0];
|
||||
const cell2 = cells[1];
|
||||
|
||||
const c1kids = cell1 ? Array.from(cell1.children) : [];
|
||||
// c1kids[0] = list (title link + specialty)
|
||||
// c1kids[1] = "Job"
|
||||
// c1kids[2] = employment type (Full-Time / Part-Time)
|
||||
// c1kids[3] = site/company name
|
||||
|
||||
const employmentType = c1kids[2]?.innerText?.trim() || null;
|
||||
const company = c1kids[3]?.innerText?.trim() || null;
|
||||
|
||||
const c2kids = cell2 ? Array.from(cell2.children) : [];
|
||||
// c2kids[0] = programs + status
|
||||
// c2kids[1] = location
|
||||
const location = c2kids[1]?.innerText?.trim() || null;
|
||||
|
||||
return {
|
||||
title: a.innerText.trim(),
|
||||
url: a.href,
|
||||
company,
|
||||
location,
|
||||
employment_type: employmentType,
|
||||
};
|
||||
}).filter(j => j && j.title && j.url.includes('/opportunity/'));
|
||||
}""")
|
||||
|
||||
cards = []
|
||||
for item in raw_data:
|
||||
cards.append(RawJobCard(
|
||||
title=item["title"],
|
||||
company=item["company"],
|
||||
location=item["location"],
|
||||
employment_type=item["employment_type"],
|
||||
url=item["url"],
|
||||
))
|
||||
|
||||
logger.info(f"[hrsa] Extracted {len(cards)} cards")
|
||||
return cards
|
||||
|
||||
def paginate(self, page, page_index: int, config) -> bool:
|
||||
"""Click 'Go to next page' button if available (client-side SPA pagination)."""
|
||||
next_btn = page.query_selector('button[aria-label="Go to next page"]')
|
||||
if not next_btn or not next_btn.is_enabled():
|
||||
return False
|
||||
|
||||
next_btn.click()
|
||||
try:
|
||||
page.wait_for_load_state("networkidle", timeout=15000)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
logger.debug(f"[hrsa] Navigated to page {page_index + 2}")
|
||||
return True
|
||||
|
||||
def normalize(self, raw: RawJobCard) -> JobPostingCandidate:
|
||||
return super().normalize(raw)
|
||||
@ -0,0 +1,50 @@
|
||||
site_id: hrsa
|
||||
enabled: true
|
||||
repair_needed: false
|
||||
|
||||
identity:
|
||||
label: HRSA Health Workforce Connector
|
||||
category: government
|
||||
base_url: https://connector.hrsa.gov
|
||||
start_url_template: "https://connector.hrsa.gov/connector/search"
|
||||
|
||||
browser:
|
||||
profile_mode: persistent_chrome_profile
|
||||
profile_name: JobAgent
|
||||
headed_on_learn: true
|
||||
headless_on_run: true
|
||||
|
||||
search:
|
||||
keyword_mode: none
|
||||
location_mode: none
|
||||
sort_mode: none
|
||||
date_mode: none
|
||||
result_list_wait_selector: "a[href*='/site-profile/'][href*='/opportunity/']"
|
||||
|
||||
search_override:
|
||||
keywords:
|
||||
- dentist
|
||||
|
||||
pagination:
|
||||
mode: none
|
||||
max_pages: 10
|
||||
|
||||
extract:
|
||||
container_selectors:
|
||||
- "a[href*='/site-profile/'][href*='/opportunity/']"
|
||||
fields:
|
||||
title:
|
||||
text: []
|
||||
|
||||
post_filters:
|
||||
include_posted_text: []
|
||||
|
||||
healthcheck:
|
||||
min_items_expected: 0
|
||||
required_fields:
|
||||
- title
|
||||
fail_if_zero_when_known_active: false
|
||||
|
||||
learn:
|
||||
last_learned_at: null
|
||||
source_url: null
|
||||
Loading…
Reference in New Issue