From 83fb8542a9ecf57c6847f85a935b07778190f13e Mon Sep 17 00:00:00 2001 From: I Luk Kim Date: Fri, 27 Mar 2026 23:16:57 -0700 Subject: [PATCH] Add 8 site adapters, early stop, Telegram notifications, and list command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New site adapters: - usajobs: multi-keyword sweep (dentist/orthodontics/orthodontist), URL pagination - docshealth, southernortho: Paylocity platform, keyword in URL - pdshealth, saltdental: iCIMS Angular platform, URL pagination - hospitaljobsonline: Load More button, relative URL fix - aaoinfo: AAO Career Center, click-based AJAX pagination - aroragroup: Load More + post-scrape keyword filter Core improvements: - Early stop pagination: stops when all fingerprints on a page are already in DB - multi_keyword_mode: separate — one scrape per keyword, cross-sweep dedup - gimme-job list command to view collected postings - Stored column in status command - Telegram notification (replacing KakaoTalk) - DB repo: count_by_site(), get_existing_fingerprints() Co-Authored-By: Claude Sonnet 4.6 --- gimme_job/adapters/aaoinfo.py | 59 +++++++++++++ gimme_job/adapters/aroragroup.py | 68 ++++++++++++++ gimme_job/adapters/docshealth.py | 34 +++++++ gimme_job/adapters/hospitaljobsonline.py | 65 ++++++++++++++ gimme_job/adapters/pdshealth.py | 48 ++++++++++ gimme_job/adapters/saltdental.py | 46 ++++++++++ gimme_job/adapters/southernortho.py | 28 ++++++ gimme_job/adapters/usajobs.py | 58 ++++++++++++ gimme_job/cli.py | 107 ++++++++++++++++++++++- gimme_job/config.py | 2 +- gimme_job/db/repo.py | 18 ++++ gimme_job/models/manifest.py | 1 + gimme_job/runtime/browser.py | 37 +++++++- gimme_job/runtime/notifier.py | 20 ++--- gimme_job/runtime/orchestrator.py | 104 +++++++++++++++++----- gimme_job/runtime/telegram.py | 53 +++++++++++ pyproject.toml | 1 + sites/aaoinfo.yaml | 60 +++++++++++++ sites/aroragroup.yaml | 56 ++++++++++++ sites/docshealth.yaml | 64 ++++++++++++++ sites/global.yaml | 2 +- sites/hospitaljobsonline.yaml | 60 +++++++++++++ sites/indeed.yaml | 4 +- sites/linkedin.yaml | 74 ++++++++++++++++ sites/pdshealth.yaml | 57 ++++++++++++ sites/saltdental.yaml | 57 ++++++++++++ sites/southernortho.yaml | 57 ++++++++++++ sites/usajobs.yaml | 70 +++++++++++++++ uv.lock | 14 +++ 29 files changed, 1283 insertions(+), 41 deletions(-) create mode 100644 gimme_job/adapters/aaoinfo.py create mode 100644 gimme_job/adapters/aroragroup.py create mode 100644 gimme_job/adapters/docshealth.py create mode 100644 gimme_job/adapters/hospitaljobsonline.py create mode 100644 gimme_job/adapters/pdshealth.py create mode 100644 gimme_job/adapters/saltdental.py create mode 100644 gimme_job/adapters/southernortho.py create mode 100644 gimme_job/adapters/usajobs.py create mode 100644 gimme_job/runtime/telegram.py create mode 100644 sites/aaoinfo.yaml create mode 100644 sites/aroragroup.yaml create mode 100644 sites/docshealth.yaml create mode 100644 sites/hospitaljobsonline.yaml create mode 100644 sites/linkedin.yaml create mode 100644 sites/pdshealth.yaml create mode 100644 sites/saltdental.yaml create mode 100644 sites/southernortho.yaml create mode 100644 sites/usajobs.yaml diff --git a/gimme_job/adapters/aaoinfo.py b/gimme_job/adapters/aaoinfo.py new file mode 100644 index 0000000..63ef908 --- /dev/null +++ b/gimme_job/adapters/aaoinfo.py @@ -0,0 +1,59 @@ +"""AAO Career Center adapter — click-based AJAX pagination.""" +from __future__ import annotations + +from loguru import logger + +from gimme_job.adapters.base import ManifestDrivenAdapter +from gimme_job.adapters.registry import register +from gimme_job.models.dto import JobPostingCandidate, RawJobCard + + +@register("aaoinfo") +class AAOInfoAdapter(ManifestDrivenAdapter): + _BASE_URL = "https://careers.aaoinfo.org" + + def _wait_for_results(self, page) -> None: + try: + page.wait_for_selector("div.job-tile", timeout=15000, state="attached") + except Exception: + logger.debug("[aaoinfo] Results wait timed out — may be zero results") + + def collect_cards(self, page, manifest): + self._wait_for_results(page) + return super().collect_cards(page, manifest) + + def paginate(self, page, page_index: int, config) -> bool: + """Click the next page number button in the AJAX pagination.""" + if page_index >= config.pagination.max_pages - 1: + return False + + next_page = page_index + 2 + + # Try clicking the specific page number button + next_btn = page.query_selector(f"#page-item-{next_page} a") + if not next_btn: + # Fall back to the last non-active/non-disabled pagination link + next_btn = page.query_selector( + "ul.pagination li.page-item:not(.active):not(.disabled):last-child a" + ) + + if not next_btn: + logger.debug(f"[aaoinfo] No next page button found — stopping at page {page_index + 1}") + return False + + logger.debug(f"[aaoinfo] Paginating to page {next_page}") + next_btn.click() + + # Wait for the new page's active indicator to confirm AJAX completed + try: + page.wait_for_selector(f"#page-item-{next_page}.active", timeout=12000) + except Exception: + page.wait_for_timeout(3000) + + return True + + def normalize(self, raw: RawJobCard) -> JobPostingCandidate: + candidate = super().normalize(raw) + if candidate.job_url and candidate.job_url.startswith("/"): + candidate.job_url = self._BASE_URL + candidate.job_url + return candidate diff --git a/gimme_job/adapters/aroragroup.py b/gimme_job/adapters/aroragroup.py new file mode 100644 index 0000000..12d59a9 --- /dev/null +++ b/gimme_job/adapters/aroragroup.py @@ -0,0 +1,68 @@ +"""Arora Group Jobs adapter — Load More button, keyword post-filter.""" +from __future__ import annotations + +from loguru import logger + +from gimme_job.adapters.base import ManifestDrivenAdapter +from gimme_job.adapters.registry import register +from gimme_job.models.dto import JobPostingCandidate, RawJobCard + +_KEYWORDS = ["dentist", "orthodontic", "orthodontist"] +_MAX_LOAD_MORE = 3 # clicks in addition to the initial load + + +@register("aroragroup") +class AroraGroupAdapter(ManifestDrivenAdapter): + _BASE_URL = "https://jobs.aroragroup.com" + + def collect_cards(self, page, manifest): + """Load initial results, click 'Load More' up to N times, then filter by keywords.""" + try: + page.wait_for_selector(".job-post-row", timeout=15000, state="attached") + except Exception: + logger.debug("[aroragroup] Initial results wait timed out") + + for i in range(_MAX_LOAD_MORE): + btn = page.query_selector("button#loadMore") + if not btn or not btn.is_visible(): + logger.debug(f"[aroragroup] Load More button gone after {i} extra loads") + break + + current_count = len(page.query_selector_all(".job-post-row[onclick]")) + btn.click() + + try: + page.wait_for_function( + f"document.querySelectorAll('.job-post-row[onclick]').length > {current_count}", + timeout=15000, + ) + logger.debug(f"[aroragroup] Load More {i + 1}/{_MAX_LOAD_MORE}: loaded more cards") + except Exception: + logger.debug(f"[aroragroup] Load More {i + 1}: no new cards appeared — stopping") + break + + all_cards = super().collect_cards(page, manifest) + + # Filter by keywords in title + filtered = [c for c in all_cards if self._matches_keywords(c.title)] + logger.debug( + f"[aroragroup] Keyword filter: {len(filtered)}/{len(all_cards)} cards matched" + ) + return filtered + + def paginate(self, page, page_index: int, config) -> bool: + """Pagination is handled inside collect_cards — always return False.""" + return False + + def normalize(self, raw: RawJobCard) -> JobPostingCandidate: + candidate = super().normalize(raw) + if candidate.job_url and candidate.job_url.startswith("/"): + candidate.job_url = self._BASE_URL + candidate.job_url + return candidate + + @staticmethod + def _matches_keywords(title: str | None) -> bool: + if not title: + return False + title_lower = title.lower() + return any(kw in title_lower for kw in _KEYWORDS) diff --git a/gimme_job/adapters/docshealth.py b/gimme_job/adapters/docshealth.py new file mode 100644 index 0000000..e9c1d9c --- /dev/null +++ b/gimme_job/adapters/docshealth.py @@ -0,0 +1,34 @@ +"""DOCS Health adapter — Paylocity-hosted job board, single-page results.""" +from __future__ import annotations + +from loguru import logger + +from gimme_job.adapters.base import ManifestDrivenAdapter +from gimme_job.adapters.registry import register +from gimme_job.models.dto import JobPostingCandidate, RawJobCard + + +@register("docshealth") +class DocsHealthAdapter(ManifestDrivenAdapter): + _BASE_URL = "https://recruiting.paylocity.com" + _COMPANY = "DOCS Health" + + def collect_cards(self, page, manifest): + # Wait for job listings to render + try: + page.wait_for_selector(".job-listing-job-item", timeout=12000, state="attached") + except Exception: + logger.debug("[docshealth] Results wait timed out — may be zero results") + return super().collect_cards(page, manifest) + + def normalize(self, raw: RawJobCard) -> JobPostingCandidate: + candidate = super().normalize(raw) + # Fix relative URL → absolute + if candidate.job_url and candidate.job_url.startswith("/"): + candidate.job_url = self._BASE_URL + candidate.job_url + # Company is always DOCS Health (not in per-card HTML) + candidate.company = self._COMPANY + # Clean posted_text: "03/27/2026 - " → "03/27/2026" + if candidate.posted_text: + candidate.posted_text = candidate.posted_text.replace(" - ", "").strip() + return candidate diff --git a/gimme_job/adapters/hospitaljobsonline.py b/gimme_job/adapters/hospitaljobsonline.py new file mode 100644 index 0000000..feb0cc0 --- /dev/null +++ b/gimme_job/adapters/hospitaljobsonline.py @@ -0,0 +1,65 @@ +"""Hospital Jobs Online adapter — URL-based pagination with page= parameter.""" +from __future__ import annotations + +import re + +from loguru import logger + +from gimme_job.adapters.base import ManifestDrivenAdapter +from gimme_job.adapters.registry import register +from gimme_job.models.dto import JobPostingCandidate, RawJobCard + + +@register("hospitaljobsonline") +class HospitalJobsOnlineAdapter(ManifestDrivenAdapter): + _BASE_URL = "https://www.hospitaljobsonline.com" + + def _wait_for_results(self, page) -> None: + try: + page.wait_for_selector("div.jobresult", timeout=15000, state="attached") + except Exception: + logger.debug("[hospitaljobsonline] Results wait timed out — may be zero results") + + def collect_cards(self, page, manifest): + self._wait_for_results(page) + return super().collect_cards(page, manifest) + + def paginate(self, page, page_index: int, config) -> bool: + """Navigate to next page by incrementing the ?page= URL parameter.""" + if page_index >= config.pagination.max_pages - 1: + return False + + current_url = page.url + next_page = page_index + 2 + + if re.search(r"[?&]page=\d+", current_url): + next_url = re.sub(r"(page=)\d+", f"page={next_page}", current_url) + else: + sep = "&" if "?" in current_url else "?" + next_url = f"{current_url}{sep}page={next_page}" + + logger.debug(f"[hospitaljobsonline] Paginating to page {next_page}: {next_url}") + page.goto(next_url, timeout=30000) + self._wait_for_results(page) + return True + + def normalize(self, raw: RawJobCard) -> JobPostingCandidate: + candidate = super().normalize(raw) + + if candidate.job_url and candidate.job_url.startswith("/"): + candidate.job_url = self._BASE_URL + candidate.job_url + + # Parse location text: "1 week ago - Company Name - Location: City, ST" + if candidate.location: + loc = candidate.location + # Extract part after "Location:" if present + m = re.search(r"Location:\s*(.+)$", loc, re.IGNORECASE) + if m: + candidate.location = m.group(1).strip() + else: + # Fallback: take last segment after " - " + parts = [p.strip() for p in loc.split(" - ")] + if len(parts) >= 2: + candidate.location = parts[-1] + + return candidate diff --git a/gimme_job/adapters/pdshealth.py b/gimme_job/adapters/pdshealth.py new file mode 100644 index 0000000..899c514 --- /dev/null +++ b/gimme_job/adapters/pdshealth.py @@ -0,0 +1,48 @@ +"""PDS Health adapter — Angular Material job board, URL-based pagination.""" +from __future__ import annotations + +import re + +from loguru import logger + +from gimme_job.adapters.base import ManifestDrivenAdapter +from gimme_job.adapters.registry import register +from gimme_job.models.dto import JobPostingCandidate, RawJobCard + + +@register("pdshealth") +class PdsHealthAdapter(ManifestDrivenAdapter): + + def _wait_for_results(self, page) -> None: + try: + page.wait_for_selector( + "mat-expansion-panel.search-result-item", + timeout=15000, + state="attached", + ) + except Exception: + logger.debug("[pdshealth] Results wait timed out — may be zero results") + + def collect_cards(self, page, manifest): + self._wait_for_results(page) + return super().collect_cards(page, manifest) + + def paginate(self, page, page_index: int, config) -> bool: + """Navigate to next page by incrementing the &page= URL parameter.""" + max_pages = config.pagination.max_pages + if page_index >= max_pages - 1: + return False + + current_url = page.url + next_page = page_index + 2 # 0-based index → 1-based page param + + if re.search(r"[?&]page=\d+", current_url): + next_url = re.sub(r"(page=)\d+", f"page={next_page}", current_url) + else: + sep = "&" if "?" in current_url else "?" + next_url = f"{current_url}{sep}page={next_page}" + + logger.debug(f"[pdshealth] Paginating to page {next_page}: {next_url}") + page.goto(next_url, timeout=30000) + self._wait_for_results(page) + return True diff --git a/gimme_job/adapters/saltdental.py b/gimme_job/adapters/saltdental.py new file mode 100644 index 0000000..b34c8d3 --- /dev/null +++ b/gimme_job/adapters/saltdental.py @@ -0,0 +1,46 @@ +"""Salt Dental Partners adapter — iCIMS Angular platform, URL-based pagination.""" +from __future__ import annotations + +import re + +from loguru import logger + +from gimme_job.adapters.base import ManifestDrivenAdapter +from gimme_job.adapters.registry import register + + +@register("saltdental") +class SaltDentalAdapter(ManifestDrivenAdapter): + + def _wait_for_results(self, page) -> None: + try: + page.wait_for_selector( + "mat-expansion-panel.search-result-item", + timeout=15000, + state="attached", + ) + except Exception: + logger.debug("[saltdental] Results wait timed out — may be zero results") + + def collect_cards(self, page, manifest): + self._wait_for_results(page) + return super().collect_cards(page, manifest) + + def paginate(self, page, page_index: int, config) -> bool: + """Navigate to next page by incrementing the &page= URL parameter.""" + if page_index >= config.pagination.max_pages - 1: + return False + + current_url = page.url + next_page = page_index + 2 + + if re.search(r"[?&]page=\d+", current_url): + next_url = re.sub(r"(page=)\d+", f"page={next_page}", current_url) + else: + sep = "&" if "?" in current_url else "?" + next_url = f"{current_url}{sep}page={next_page}" + + logger.debug(f"[saltdental] Paginating to page {next_page}: {next_url}") + page.goto(next_url, timeout=30000) + self._wait_for_results(page) + return True diff --git a/gimme_job/adapters/southernortho.py b/gimme_job/adapters/southernortho.py new file mode 100644 index 0000000..3767120 --- /dev/null +++ b/gimme_job/adapters/southernortho.py @@ -0,0 +1,28 @@ +"""Southern Orthodontic Partners adapter — Paylocity platform, single page.""" +from __future__ import annotations + +from loguru import logger + +from gimme_job.adapters.base import ManifestDrivenAdapter +from gimme_job.adapters.registry import register +from gimme_job.models.dto import JobPostingCandidate, RawJobCard + + +@register("southernortho") +class SouthernOrthoAdapter(ManifestDrivenAdapter): + _BASE_URL = "https://recruiting.paylocity.com" + + def collect_cards(self, page, manifest): + try: + page.wait_for_selector(".job-listing-job-item", timeout=12000, state="attached") + except Exception: + logger.debug("[southernortho] Results wait timed out — may be zero results") + return super().collect_cards(page, manifest) + + def normalize(self, raw: RawJobCard) -> JobPostingCandidate: + candidate = super().normalize(raw) + if candidate.job_url and candidate.job_url.startswith("/"): + candidate.job_url = self._BASE_URL + candidate.job_url + if candidate.posted_text: + candidate.posted_text = candidate.posted_text.replace(" - ", "").strip() + return candidate diff --git a/gimme_job/adapters/usajobs.py b/gimme_job/adapters/usajobs.py new file mode 100644 index 0000000..a5bca25 --- /dev/null +++ b/gimme_job/adapters/usajobs.py @@ -0,0 +1,58 @@ +"""USAJOBS adapter — waits for JS render, handles relative URLs, URL-based pagination.""" +from __future__ import annotations + +import re + +from loguru import logger + +from gimme_job.adapters.base import ManifestDrivenAdapter +from gimme_job.adapters.registry import register +from gimme_job.models.dto import JobPostingCandidate, RawJobCard + + +@register("usajobs") +class UsaJobsAdapter(ManifestDrivenAdapter): + _BASE_URL = "https://www.usajobs.gov" + + def _wait_for_results(self, page) -> None: + """Wait for JS-rendered results or zero-results heading.""" + try: + page.wait_for_selector( + "#search-results .border, h2.font-normal", + timeout=12000, + state="attached", + ) + except Exception: + logger.debug("[usajobs] Results wait timed out — may be zero results") + + def collect_cards(self, page, manifest): + self._wait_for_results(page) + return super().collect_cards(page, manifest) + + def paginate(self, page, page_index: int, config) -> bool: + """Navigate to next page by incrementing the &p= URL parameter.""" + max_pages = config.pagination.max_pages + if page_index >= max_pages - 1: + return False + + current_url = page.url + next_page = page_index + 2 # page_index is 0-based; URL param is 1-based + + # Replace &p=N or add &p=N + if re.search(r"[?&]p=\d+", current_url): + next_url = re.sub(r"(p=)\d+", f"p={next_page}", current_url) + else: + sep = "&" if "?" in current_url else "?" + next_url = f"{current_url}{sep}p={next_page}" + + logger.debug(f"[usajobs] Paginating to page {next_page}: {next_url}") + page.goto(next_url, timeout=30000) + self._wait_for_results(page) + return True + + def normalize(self, raw: RawJobCard) -> JobPostingCandidate: + candidate = super().normalize(raw) + # USAJOBS job links are relative: /job/123456 → absolute URL + if candidate.job_url and candidate.job_url.startswith("/"): + candidate.job_url = self._BASE_URL + candidate.job_url + return candidate diff --git a/gimme_job/cli.py b/gimme_job/cli.py index 8627e7c..a6e584f 100644 --- a/gimme_job/cli.py +++ b/gimme_job/cli.py @@ -94,13 +94,14 @@ def _write_default_global_yaml(path: Path) -> None: "max_input_items": 200, }, "notification": { - "provider": "kakaotalk", + "provider": "telegram", "fallback_markdown": True, }, } write_yaml(path, data) + def _check_playwright() -> None: try: from playwright.sync_api import sync_playwright @@ -141,6 +142,44 @@ def _check_claude() -> None: console.print("[yellow]![/yellow] Claude Code CLI not found (needed for learn/repair)") +# ── login ───────────────────────────────────────────────────────────────────── + + +@app.command() +def login() -> None: + """Open the JobAgent browser to log in to sites manually. + + Browse to any sites (LinkedIn, Indeed, Google, etc.), log in, then press Enter. + All cookies and localStorage are saved in the JobAgent profile and reused on + every subsequent run. + """ + from gimme_job.config import load_global_config + from gimme_job.runtime.browser import BrowserManager + + cfg = load_global_config() + + console.print(Panel("[bold cyan]gimme-job login[/bold cyan]", expand=False)) + console.print( + "Opening JobAgent Chrome profile.\n" + "Log in to any sites you need, then press Enter here to close the browser.\n" + "All cookies and localStorage will be saved and reused on future runs.\n" + ) + + bm = BrowserManager( + profile_name=cfg.runtime.profile_name, + headless=False, + slow_mo=cfg.runtime.slow_mo_ms, + ) + try: + bm.open_context() + bm.new_page() + input(" >> Press Enter when done... ") + finally: + bm.close() + + console.print("[green]✓[/green] Browser closed. Session saved to JobAgent profile.") + + # ── run ─────────────────────────────────────────────────────────────────────── @@ -148,7 +187,7 @@ def _check_claude() -> None: def run( site: Optional[str] = typer.Option(None, "--site", help="Run a single site only"), dry_run: bool = typer.Option(False, "--dry-run", help="Extract but do not save or notify"), - skip_notify: bool = typer.Option(False, "--skip-notify", help="Skip KakaoTalk notification"), + skip_notify: bool = typer.Option(False, "--skip-notify", help="Skip Telegram notification"), today_only: bool = typer.Option(False, "--today-only", help="Only process today's new items"), limit_sites: Optional[int] = typer.Option(None, "--limit-sites", help="Max sites to run"), ) -> None: @@ -290,7 +329,7 @@ def test( def notify( today: bool = typer.Option(True, "--today/--no-today", help="Re-send today's digest"), ) -> None: - """Re-send today's job summary via KakaoTalk.""" + """Re-send today's job summary via Telegram.""" from datetime import date from gimme_job.config import load_global_config @@ -331,6 +370,59 @@ def notify( console.print("[yellow]![/yellow] Notification failed — saved as markdown fallback") +# ── list ────────────────────────────────────────────────────────────────────── + + +@app.command(name="list") +def list_postings( + site: Optional[str] = typer.Option(None, "--site", help="Filter by site ID"), + today: bool = typer.Option(False, "--today", help="Only show today's new postings"), + limit: int = typer.Option(50, "--limit", help="Max rows to show"), +) -> None: + """List collected job postings from the database.""" + from datetime import date + + from gimme_job.db.engine import get_engine, get_session_factory + from gimme_job.db.repo import JobPostingRepo + from gimme_job.models.db import JobPosting + from sqlalchemy import select + + engine = get_engine() + factory = get_session_factory(engine) + + table = Table(title="Job Postings", show_header=True, show_lines=False) + table.add_column("#", style="dim", justify="right", width=4) + table.add_column("Site", style="cyan", width=10) + table.add_column("Title", width=40) + table.add_column("Company", width=25) + table.add_column("Location", width=20) + table.add_column("Posted", width=12) + table.add_column("New", width=4) + + with factory() as session: + q = select(JobPosting).order_by(JobPosting.first_seen_at.desc()) + if site: + q = q.where(JobPosting.site_id == site) + if today: + q = q.where(JobPosting.run_date == date.today(), JobPosting.is_new == True) + q = q.limit(limit) + rows = list(session.execute(q).scalars().all()) + + for i, p in enumerate(rows, 1): + table.add_row( + str(i), + p.site_id, + p.title[:38] if p.title else "", + (p.company or "")[:23], + (p.location or "")[:18], + p.posted_text or "", + "[green]Y[/green]" if p.is_new else "", + ) + + console.print(table) + console.print(f"[dim]{len(rows)} rows (limit={limit})[/dim]") + + # ── status ──────────────────────────────────────────────────────────────────── @@ -344,10 +436,14 @@ def status() -> None: engine = get_engine() factory = get_session_factory(engine) + from sqlalchemy import func, select + from gimme_job.models.db import JobPosting + table = Table(title="Site Status", show_header=True) table.add_column("Site", style="cyan") table.add_column("Enabled") table.add_column("Repair?") + table.add_column("Stored", justify="right") table.add_column("Last Run") table.add_column("Last Status") table.add_column("Failures", justify="right") @@ -364,6 +460,9 @@ def status() -> None: config = session.get(__import__("gimme_job.models.db", fromlist=["SiteConfig"]).SiteConfig, site_id) recent = run_repo.get_recent_runs(session, site_id, limit=1) + stored = session.execute( + select(func.count()).where(JobPosting.site_id == site_id) + ).scalar() or 0 enabled = "[green]yes[/green]" if manifest.enabled else "[dim]no[/dim]" repair = "[red]YES[/red]" if manifest.repair_needed else "[green]no[/green]" @@ -371,7 +470,7 @@ def status() -> None: last_status = recent[0].status if recent else "[dim]-[/dim]" failures = str(config.consecutive_failures) if config else "0" - table.add_row(site_id, enabled, repair, last_run, last_status, failures) + table.add_row(site_id, enabled, repair, str(stored), last_run, last_status, failures) console.print(table) diff --git a/gimme_job/config.py b/gimme_job/config.py index 18d2907..92680af 100644 --- a/gimme_job/config.py +++ b/gimme_job/config.py @@ -40,7 +40,7 @@ class SummarizationConfig(BaseModel): class NotificationConfig(BaseModel): - provider: str = "kakaotalk" + provider: str = "telegram" fallback_markdown: bool = True diff --git a/gimme_job/db/repo.py b/gimme_job/db/repo.py index 5c2cb16..19f746c 100644 --- a/gimme_job/db/repo.py +++ b/gimme_job/db/repo.py @@ -88,6 +88,24 @@ class JobPostingRepo: ).scalars().all() ) + def count_by_site(self, session: Session, site_id: str) -> int: + """Count total job postings stored for a site.""" + from sqlalchemy import func + return session.execute( + select(func.count()).where(JobPosting.site_id == site_id) + ).scalar() or 0 + + def get_existing_fingerprints( + self, session: Session, fingerprints: list[str] + ) -> set[str]: + """Return the subset of given fingerprints that already exist in the DB.""" + if not fingerprints: + return set() + rows = session.execute( + select(JobPosting.fingerprint).where(JobPosting.fingerprint.in_(fingerprints)) + ).scalars().all() + return set(rows) + class SiteRunRepo: def record_run(self, session: Session, result: SiteRunResult) -> SiteRun: diff --git a/gimme_job/models/manifest.py b/gimme_job/models/manifest.py index d8a6ce6..43daabb 100644 --- a/gimme_job/models/manifest.py +++ b/gimme_job/models/manifest.py @@ -70,6 +70,7 @@ class SearchOverride(BaseModel): date_mode: Optional[str] = None sort: Optional[str] = None max_items: Optional[int] = None + multi_keyword_mode: Optional[str] = None # "separate" → one scrape per keyword class LearnMeta(BaseModel): diff --git a/gimme_job/runtime/browser.py b/gimme_job/runtime/browser.py index 4bac0e0..ddb6749 100644 --- a/gimme_job/runtime/browser.py +++ b/gimme_job/runtime/browser.py @@ -10,7 +10,12 @@ from loguru import logger class BrowserManager: """Manages Playwright browser contexts using persistent Chrome profiles.""" - def __init__(self, profile_name: str = "JobAgent", headless: bool = True, slow_mo: int = 0): + def __init__( + self, + profile_name: str = "JobAgent", + headless: bool = True, + slow_mo: int = 0, + ): self.profile_name = profile_name self.headless = headless self.slow_mo = slow_mo @@ -34,7 +39,6 @@ class BrowserManager: profile_path = chrome_profile_dir(name) profile_path.mkdir(parents=True, exist_ok=True) - logger.info(f"Opening browser context: profile={name}, headless={hl}") self._playwright = sync_playwright().start() @@ -46,12 +50,41 @@ class BrowserManager: args=[ "--disable-blink-features=AutomationControlled", "--no-sandbox", + "--disable-infobars", + "--disable-dev-shm-usage", ], + ignore_default_args=["--enable-automation"], viewport={"width": 1280, "height": 900}, locale="en-US", + user_agent=( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/134.0.0.0 Safari/537.36" + ), ) + # Apply stealth patches to every new page + try: + from playwright_stealth import stealth_sync + self._context.add_init_script(""" + Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); + """) + # stealth_sync works per-page; store reference for use in open_page() + self._stealth_fn = stealth_sync + except ImportError: + self._stealth_fn = None + return self._context + def new_page(self): + """Open a new page with stealth applied.""" + page = self._context.new_page() + if getattr(self, "_stealth_fn", None): + try: + self._stealth_fn(page) + except Exception as e: + logger.debug(f"Stealth apply failed: {e}") + return page + def close(self) -> None: if self._context: try: diff --git a/gimme_job/runtime/notifier.py b/gimme_job/runtime/notifier.py index ff6a0a3..4d753b6 100644 --- a/gimme_job/runtime/notifier.py +++ b/gimme_job/runtime/notifier.py @@ -1,4 +1,4 @@ -"""Notification dispatcher: KakaoTalk with Markdown fallback.""" +"""Notification dispatcher: Telegram with Markdown fallback.""" from __future__ import annotations from datetime import date @@ -15,25 +15,23 @@ class NotificationDispatcher: def send(self, summary: str, run_date: date) -> bool: """Send the summary. Returns True if any method succeeded.""" - from gimme_job.db.repo import NotificationLogRepo - from gimme_job.runtime.kakao import KakaoTalkClient + from gimme_job.runtime.telegram import TelegramClient - client = KakaoTalkClient() + client = TelegramClient() success = False if client.is_configured(): - ok = client.send_self_memo(summary) + ok = client.send_message(summary) if ok: success = True - self._log_notification(run_date, "kakaotalk", "success") + self._log_notification(run_date, "telegram", "success") else: - logger.warning("KakaoTalk failed — writing markdown fallback") - self._log_notification(run_date, "kakaotalk", "failed", "send_self_memo returned False") + logger.warning("Telegram failed — writing markdown fallback") + self._log_notification(run_date, "telegram", "failed", "send_message returned False") else: - logger.warning("KakaoTalk not configured — writing markdown fallback only") - self._log_notification(run_date, "kakaotalk", "skipped", "not configured") + logger.warning("Telegram not configured — writing markdown fallback only") + self._log_notification(run_date, "telegram", "skipped", "not configured") - # Always save markdown fallback if configured if self.cfg.notification.fallback_markdown: path = self.send_markdown_fallback(summary, run_date) logger.info(f"Markdown saved: {path}") diff --git a/gimme_job/runtime/orchestrator.py b/gimme_job/runtime/orchestrator.py index 2bd0d42..b3addab 100644 --- a/gimme_job/runtime/orchestrator.py +++ b/gimme_job/runtime/orchestrator.py @@ -56,9 +56,10 @@ class RunOrchestrator: # Record to DB (unless dry run) if not dry_run: - with self.session_factory() as session: - from gimme_job.db.repo import SiteRunRepo - from gimme_job.runtime.health import update_site_status + from gimme_job.db.engine import db_session + from gimme_job.db.repo import SiteRunRepo + from gimme_job.runtime.health import update_site_status + with db_session() as session: SiteRunRepo().record_run(session, result) update_site_status(site_id, result, session) @@ -97,7 +98,21 @@ class RunOrchestrator: from gimme_job.utils.paths import screenshot_path, dom_snapshot_path, trace_path try: - candidates = self._scrape_site(site_id, manifest, query, run_id, result) + if manifest.search_override.multi_keyword_mode == "separate" and query.keywords: + # Run one scrape per keyword and merge results + from gimme_job.runtime.dedupe import deduplicate_in_batch + all_candidates = [] + for kw in query.keywords: + from gimme_job.models.dto import SearchQuery + kw_query = query.model_copy(update={"keywords": [kw]}) + kw_run_id = f"{run_id}-{kw[:8]}" + logger.info(f"[{site_id}] Keyword sweep: '{kw}'") + kw_candidates = self._scrape_site(site_id, manifest, kw_query, kw_run_id, result, dry_run=dry_run) + all_candidates.extend(kw_candidates) + # Deduplicate across keyword sweeps by fingerprint + candidates = deduplicate_in_batch(all_candidates) + else: + candidates = self._scrape_site(site_id, manifest, query, run_id, result, dry_run=dry_run) except Exception as e: logger.error(f"[{site_id}] Scrape error: {e}") result.status = RunStatus.FAILED @@ -108,8 +123,9 @@ class RunOrchestrator: result.items_found = len(candidates) if not dry_run and candidates: - with self.session_factory() as session: - from gimme_job.db.repo import JobPostingRepo + from gimme_job.db.engine import db_session + from gimme_job.db.repo import JobPostingRepo + with db_session() as session: total, new_count = JobPostingRepo().upsert_candidates(session, candidates, run_date) result.new_items = new_count elif dry_run: @@ -131,6 +147,7 @@ class RunOrchestrator: query: SearchQuery, run_id: str, result: SiteRunResult, + dry_run: bool = False, ): from gimme_job.adapters.registry import get_adapter from gimme_job.runtime.browser import BrowserManager @@ -141,13 +158,17 @@ class RunOrchestrator: headless = manifest.browser.headless_on_run profile = manifest.browser.profile_name or self.cfg.runtime.profile_name - bm = BrowserManager(profile_name=profile, headless=headless, slow_mo=self.cfg.runtime.slow_mo_ms) + bm = BrowserManager( + profile_name=profile, + headless=headless, + slow_mo=self.cfg.runtime.slow_mo_ms, + ) all_cards = [] try: context = bm.open_context() bm.start_tracing() - page = context.new_page() + page = bm.new_page() # Set timeouts page.set_default_timeout(self.cfg.runtime.default_timeout_ms) @@ -177,6 +198,17 @@ class RunOrchestrator: except Exception: logger.warning(f"[{site_id}] Wait selector timed out — continuing anyway") + # Check if site has existing data (for early stop on update runs) + site_has_data = False + if not dry_run: + try: + from gimme_job.db.engine import db_session + from gimme_job.db.repo import JobPostingRepo + with db_session() as _s: + site_has_data = JobPostingRepo().count_by_site(_s, site_id) > 0 + except Exception: + pass + # Collect cards across pages max_pages = min( manifest.pagination.max_pages, @@ -187,6 +219,35 @@ class RunOrchestrator: all_cards.extend(page_cards) logger.debug(f"[{site_id}] Page {page_idx+1}: {len(page_cards)} cards") + if not page_cards: + break # no results on this page — stop paginating + + # Early stop: if site has existing data and this page's cards are all known, + # we've caught up to previously stored content — no need to paginate further. + if site_has_data and page_idx >= 1: + try: + from gimme_job.db.engine import db_session + from gimme_job.db.repo import JobPostingRepo + page_fps = [] + for card in page_cards: + try: + c = adapter.normalize(card) + if c.fingerprint: + page_fps.append(c.fingerprint) + except Exception: + pass + if page_fps: + with db_session() as _s: + known = JobPostingRepo().get_existing_fingerprints(_s, page_fps) + if len(known) >= len(page_fps): + logger.info( + f"[{site_id}] Page {page_idx+1}: all {len(page_fps)} items" + " already in DB — stopping early" + ) + break + except Exception as e: + logger.debug(f"[{site_id}] Early stop check failed: {e}") + if page_idx < max_pages - 1: if not adapter.paginate(page, page_idx, manifest): break @@ -247,22 +308,25 @@ class RunOrchestrator: def _summarize_and_notify(self, summary: RunSummary, run_date: date) -> None: try: - with self.session_factory() as session: - from gimme_job.db.repo import JobPostingRepo, SummaryRepo + from gimme_job.db.engine import db_session + from gimme_job.db.repo import JobPostingRepo, SummaryRepo + + with db_session() as session: postings = JobPostingRepo().get_today_new(session, run_date) - if not postings: - return + if not postings: + return - from gimme_job.runtime.summarizer import OllamaSummarizer - summarizer = OllamaSummarizer( - base_url=self.cfg.summarization.ollama_base_url, - model=self.cfg.summarization.model, - temperature=self.cfg.summarization.temperature, - ) - text = summarizer.summarize(postings) - summary.summary_text = text + from gimme_job.runtime.summarizer import OllamaSummarizer + summarizer = OllamaSummarizer( + base_url=self.cfg.summarization.ollama_base_url, + model=self.cfg.summarization.model, + temperature=self.cfg.summarization.temperature, + ) + text = summarizer.summarize(postings) + summary.summary_text = text + with db_session() as session: SummaryRepo().save_summary(session, run_date, text, self.cfg.summarization.model) from gimme_job.runtime.notifier import NotificationDispatcher diff --git a/gimme_job/runtime/telegram.py b/gimme_job/runtime/telegram.py new file mode 100644 index 0000000..3cd9d51 --- /dev/null +++ b/gimme_job/runtime/telegram.py @@ -0,0 +1,53 @@ +"""Telegram notification client — sends messages via Bot API.""" +from __future__ import annotations + +import os +from typing import Optional + +from loguru import logger + +TELEGRAM_API_URL = "https://api.telegram.org/bot{token}/sendMessage" +MAX_TEXT_LENGTH = 4096 # Telegram message limit + + +class TelegramClient: + def __init__( + self, + bot_token: Optional[str] = None, + chat_id: Optional[str] = None, + ): + self.bot_token = bot_token or os.environ.get("TELEGRAM_BOT_TOKEN", "") + self.chat_id = chat_id or os.environ.get("TELEGRAM_CHAT_ID", "") + + def is_configured(self) -> bool: + return bool(self.bot_token and self.chat_id) + + def send_message(self, text: str) -> bool: + """Send a message to the configured chat. Returns True on success.""" + if not self.is_configured(): + logger.warning("Telegram not configured (missing BOT_TOKEN or CHAT_ID)") + return False + + if len(text) > MAX_TEXT_LENGTH: + text = text[: MAX_TEXT_LENGTH - 20] + "\n\n[...truncated]" + + try: + import httpx + response = httpx.post( + TELEGRAM_API_URL.format(token=self.bot_token), + json={ + "chat_id": self.chat_id, + "text": text, + "parse_mode": "HTML", + }, + timeout=15.0, + ) + if response.status_code == 200 and response.json().get("ok"): + logger.info("Telegram message sent successfully") + return True + else: + logger.error(f"Telegram API error: {response.text[:200]}") + return False + except Exception as e: + logger.error(f"Telegram send failed: {e}") + return False diff --git a/pyproject.toml b/pyproject.toml index 61f94d5..1599be8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ dependencies = [ "jinja2>=3.1", "pyyaml>=6.0", "python-dotenv>=1.0", + "playwright-stealth>=2.0.2", ] [project.scripts] diff --git a/sites/aaoinfo.yaml b/sites/aaoinfo.yaml new file mode 100644 index 0000000..f04cab9 --- /dev/null +++ b/sites/aaoinfo.yaml @@ -0,0 +1,60 @@ +site_id: aaoinfo +enabled: true +repair_needed: false + +identity: + label: AAO Career Center + category: aggregator + base_url: https://careers.aaoinfo.org + start_url_template: "https://careers.aaoinfo.org/jobs/" + +browser: + profile_mode: persistent_chrome_profile + profile_name: JobAgent + headed_on_learn: true + headless_on_run: true + +search: + keyword_mode: none + location_mode: none + sort_mode: none + date_mode: none + result_list_wait_selector: "div.job-tile" + +pagination: + mode: none + max_pages: 10 + +extract: + container_selectors: + - "div.job-tile:not(.job-mo)" + fields: + title: + text: + - ".job-title a" + company: + text: + - ".job-company-row" + location: + text: + - ".job-location" + posted_text: + text: + - ".job-posted-date" + url: + attr: + selector: ".job-title a" + name: href + +post_filters: + include_posted_text: [] + +healthcheck: + min_items_expected: 0 + required_fields: + - title + fail_if_zero_when_known_active: false + +learn: + last_learned_at: null + source_url: null diff --git a/sites/aroragroup.yaml b/sites/aroragroup.yaml new file mode 100644 index 0000000..2459f2f --- /dev/null +++ b/sites/aroragroup.yaml @@ -0,0 +1,56 @@ +site_id: aroragroup +enabled: true +repair_needed: false + +identity: + label: Arora Group Jobs + category: employer + base_url: https://jobs.aroragroup.com + start_url_template: "https://jobs.aroragroup.com/" + +browser: + profile_mode: persistent_chrome_profile + profile_name: JobAgent + headed_on_learn: true + headless_on_run: true + +search: + keyword_mode: none + location_mode: none + sort_mode: none + date_mode: none + result_list_wait_selector: ".job-post-row" + +pagination: + mode: none + max_pages: 1 + +extract: + container_selectors: + - ".job-post-row[onclick]" + fields: + title: + text: + - ".POST_TITLE" + - "h3.job-post-title a" + location: + text: + - ".POST_LOCATION" + - ".job-post-location" + url: + attr: + selector: "a.job-post-href" + name: href + +post_filters: + include_posted_text: [] + +healthcheck: + min_items_expected: 0 + required_fields: + - title + fail_if_zero_when_known_active: false + +learn: + last_learned_at: null + source_url: null diff --git a/sites/docshealth.yaml b/sites/docshealth.yaml new file mode 100644 index 0000000..626fdee --- /dev/null +++ b/sites/docshealth.yaml @@ -0,0 +1,64 @@ +site_id: docshealth +enabled: true +repair_needed: false + +identity: + label: DOCS Health + category: employer + base_url: https://recruiting.paylocity.com + start_url_template: "https://recruiting.paylocity.com/recruiting/jobs/All/4f5b87cb-fda9-4243-976b-499bbf0ce9fc/DOCS-Health?search={keywords_urlencoded}&location=All%20Locations&department=All%20Departments" + +browser: + profile_mode: persistent_chrome_profile + profile_name: JobAgent + headed_on_learn: true + headless_on_run: true + +search: + keyword_mode: url_param + location_mode: url_param + sort_mode: none + date_mode: none + result_list_wait_selector: ".job-listing-job-item" + +search_override: + keywords: + - dentist + - orthodontics + - orthodontist + multi_keyword_mode: separate + +pagination: + mode: none + max_pages: 1 + +extract: + container_selectors: + - "div.job-listing-job-item" + fields: + title: + text: + - ".job-item-title a" + location: + text: + - ".location-column .job-item-normal" + posted_text: + text: + - ".job-title-column > span:not(.job-item-title):not(.job-item-normal)" + url: + attr: + selector: ".job-item-title a" + name: href + +post_filters: + include_posted_text: [] + +healthcheck: + min_items_expected: 0 + required_fields: + - title + fail_if_zero_when_known_active: false + +learn: + last_learned_at: null + source_url: null diff --git a/sites/global.yaml b/sites/global.yaml index 654547e..09f1889 100644 --- a/sites/global.yaml +++ b/sites/global.yaml @@ -5,7 +5,7 @@ runtime: slow_mo_ms: 0 default_timeout_ms: 15000 navigation_timeout_ms: 30000 - max_pages_per_site: 3 + max_pages_per_site: 10 min_delay_ms: 1200 max_delay_ms: 3500 diff --git a/sites/hospitaljobsonline.yaml b/sites/hospitaljobsonline.yaml new file mode 100644 index 0000000..f7f66f9 --- /dev/null +++ b/sites/hospitaljobsonline.yaml @@ -0,0 +1,60 @@ +site_id: hospitaljobsonline +enabled: true +repair_needed: false + +identity: + label: Hospital Jobs Online + category: aggregator + base_url: https://www.hospitaljobsonline.com + start_url_template: "https://www.hospitaljobsonline.com/jobs/q-korea-jobs//?page=1&limit=100" + +browser: + profile_mode: persistent_chrome_profile + profile_name: JobAgent + headed_on_learn: true + headless_on_run: true + +search: + keyword_mode: none + location_mode: none + sort_mode: none + date_mode: none + result_list_wait_selector: "div.jobresult" + +pagination: + mode: url_increment + max_pages: 10 + +extract: + container_selectors: + - "div.jobresult" + fields: + title: + text: + - ".jobtitle a" + company: + text: + - ".location a" + location: + text: + - ".location" + posted_text: + text: + - ".location .date" + url: + attr: + selector: ".jobtitle a" + name: href + +post_filters: + include_posted_text: [] + +healthcheck: + min_items_expected: 0 + required_fields: + - title + fail_if_zero_when_known_active: false + +learn: + last_learned_at: null + source_url: null diff --git a/sites/indeed.yaml b/sites/indeed.yaml index f8707cd..3391f7e 100644 --- a/sites/indeed.yaml +++ b/sites/indeed.yaml @@ -1,5 +1,5 @@ site_id: indeed -enabled: true +enabled: false repair_needed: false identity: @@ -12,7 +12,7 @@ browser: profile_mode: persistent_chrome_profile profile_name: JobAgent headed_on_learn: true - headless_on_run: true + headless_on_run: false search: keyword_mode: url_param diff --git a/sites/linkedin.yaml b/sites/linkedin.yaml new file mode 100644 index 0000000..6be5cc5 --- /dev/null +++ b/sites/linkedin.yaml @@ -0,0 +1,74 @@ +site_id: linkedin +enabled: true +repair_needed: false + +identity: + label: LinkedIn Jobs + category: aggregator + base_url: https://www.linkedin.com + start_url_template: "https://www.linkedin.com/jobs/search/?keywords={keywords_urlencoded}&f_TPR=r86400&sortBy=DD" + +browser: + profile_mode: persistent_chrome_profile + profile_name: JobAgent + headed_on_learn: true + headless_on_run: false + +search: + keyword_mode: url_param + location_mode: url_param + sort_mode: url_param + date_mode: url_param + result_list_wait_selector: ".jobs-search__results-list li, .scaffold-layout__list-container li" + +pagination: + mode: next_button + next_button_selectors: + - "button[aria-label='View next page']" + - "button[aria-label*='next']" + - "li.artdeco-pagination__indicator--number.selected + li button" + max_pages: 3 + +extract: + container_selectors: + - ".jobs-search__results-list li" + - ".scaffold-layout__list-container li" + - "li:has(.base-card)" + fields: + title: + text: + - "h3.base-search-card__title" + - ".job-card-list__title" + - "h3" + - "a[data-control-name='job_card_title']" + company: + text: + - "h4.base-search-card__subtitle" + - ".job-card-container__primary-description" + - "a.job-card-container__company-name" + location: + text: + - "span.job-search-card__location" + - ".job-card-container__metadata-item" + posted_text: + text: + - "time" + - ".job-search-card__listdate" + - "span[class*='listdate']" + url: + attr: + selector: "a.base-card__full-link, a[data-control-name='job_card_title'], a[href*='/jobs/view/']" + name: href + +post_filters: + include_posted_text: [] + +healthcheck: + min_items_expected: 0 + required_fields: + - title + fail_if_zero_when_known_active: false + +learn: + last_learned_at: null + source_url: null diff --git a/sites/pdshealth.yaml b/sites/pdshealth.yaml new file mode 100644 index 0000000..0f477b2 --- /dev/null +++ b/sites/pdshealth.yaml @@ -0,0 +1,57 @@ +site_id: pdshealth +enabled: true +repair_needed: false + +identity: + label: PDS Health + category: employer + base_url: https://jobs.pdshealth.com + start_url_template: "https://jobs.pdshealth.com/jobs?keywords=orthodontist&sortBy=relevance&page=1&limit=100" + +browser: + profile_mode: persistent_chrome_profile + profile_name: JobAgent + headed_on_learn: true + headless_on_run: true + +search: + keyword_mode: none + location_mode: none + sort_mode: none + date_mode: none + result_list_wait_selector: "mat-expansion-panel.search-result-item" + +pagination: + mode: url_increment + max_pages: 20 + +extract: + container_selectors: + - "mat-expansion-panel.search-result-item" + fields: + title: + text: + - "a.job-title-link" + company: + text: + - ".label-value.tags1" + location: + text: + - ".label-value.location" + url: + attr: + selector: "a.job-title-link" + name: href + +post_filters: + include_posted_text: [] + +healthcheck: + min_items_expected: 0 + required_fields: + - title + fail_if_zero_when_known_active: false + +learn: + last_learned_at: null + source_url: null diff --git a/sites/saltdental.yaml b/sites/saltdental.yaml new file mode 100644 index 0000000..6706e86 --- /dev/null +++ b/sites/saltdental.yaml @@ -0,0 +1,57 @@ +site_id: saltdental +enabled: true +repair_needed: false + +identity: + label: Salt Dental Partners + category: employer + base_url: https://jobs.saltdentalpartners.com + start_url_template: "https://jobs.saltdentalpartners.com/jobs?keywords=orthodontist&sortBy=relevance&page=1" + +browser: + profile_mode: persistent_chrome_profile + profile_name: JobAgent + headed_on_learn: true + headless_on_run: true + +search: + keyword_mode: none + location_mode: none + sort_mode: none + date_mode: none + result_list_wait_selector: "mat-expansion-panel.search-result-item" + +pagination: + mode: url_increment + max_pages: 5 + +extract: + container_selectors: + - "mat-expansion-panel.search-result-item" + fields: + title: + text: + - "a.job-title-link" + company: + text: + - ".label-value.tags1" + location: + text: + - ".label-value.location" + url: + attr: + selector: "a.job-title-link" + name: href + +post_filters: + include_posted_text: [] + +healthcheck: + min_items_expected: 0 + required_fields: + - title + fail_if_zero_when_known_active: false + +learn: + last_learned_at: null + source_url: null diff --git a/sites/southernortho.yaml b/sites/southernortho.yaml new file mode 100644 index 0000000..6ac31cc --- /dev/null +++ b/sites/southernortho.yaml @@ -0,0 +1,57 @@ +site_id: southernortho +enabled: true +repair_needed: false + +identity: + label: Southern Orthodontic Partners + category: employer + base_url: https://recruiting.paylocity.com + start_url_template: "https://recruiting.paylocity.com/recruiting/jobs/All/06144481-7908-457c-9312-51cf7b58cf01/Southern-Orthodontic-Partners-Mgmt-LLC?search=orthodontist&location=All%20Locations&department=All%20Departments" + +browser: + profile_mode: persistent_chrome_profile + profile_name: JobAgent + headed_on_learn: true + headless_on_run: true + +search: + keyword_mode: none + location_mode: none + sort_mode: none + date_mode: none + result_list_wait_selector: ".job-listing-job-item" + +pagination: + mode: none + max_pages: 1 + +extract: + container_selectors: + - "div.job-listing-job-item" + fields: + title: + text: + - ".job-item-title a" + company: + text: + - ".location-column .job-item-normal" + posted_text: + text: + - ".job-title-column > span:not(.job-item-title):not(.job-item-normal)" + url: + attr: + selector: ".job-item-title a" + name: href + +post_filters: + include_posted_text: [] + +healthcheck: + min_items_expected: 0 + required_fields: + - title + fail_if_zero_when_known_active: false + +learn: + last_learned_at: null + source_url: null diff --git a/sites/usajobs.yaml b/sites/usajobs.yaml new file mode 100644 index 0000000..ab6396e --- /dev/null +++ b/sites/usajobs.yaml @@ -0,0 +1,70 @@ +site_id: usajobs +enabled: true +repair_needed: false + +identity: + label: USAJOBS + category: government + base_url: https://www.usajobs.gov + start_url_template: "https://www.usajobs.gov/Search/Results?k={keywords_urlencoded}&p=1" + +browser: + profile_mode: persistent_chrome_profile + profile_name: JobAgent + headed_on_learn: true + headless_on_run: true + +search: + keyword_mode: url_param + location_mode: url_param + sort_mode: url_param + date_mode: url_param + result_list_wait_selector: "" + +pagination: + mode: next_button + next_button_selectors: + - "button#page-m-next" + - "button[title='Go to next page']" + max_pages: 10 + +extract: + container_selectors: + - "#search-results .border.border-gray-lighter" + fields: + title: + text: + - "h2 a" + company: + text: + - "p strong" + location: + text: + - "div:nth-child(2) div:first-child p:nth-child(3)" + posted_text: + text: + - ".italic" + url: + attr: + selector: "h2 a" + name: href + +search_override: + keywords: + - dentist + - orthodontics + - orthodontist + multi_keyword_mode: separate + +post_filters: + include_posted_text: [] + +healthcheck: + min_items_expected: 0 + required_fields: + - title + fail_if_zero_when_known_active: false + +learn: + last_learned_at: null + source_url: null diff --git a/uv.lock b/uv.lock index 876ee21..94c7ffc 100644 --- a/uv.lock +++ b/uv.lock @@ -156,6 +156,7 @@ dependencies = [ { name = "jinja2" }, { name = "loguru" }, { name = "playwright" }, + { name = "playwright-stealth" }, { name = "pydantic" }, { name = "pydantic-settings" }, { name = "python-dotenv" }, @@ -179,6 +180,7 @@ requires-dist = [ { name = "jinja2", specifier = ">=3.1" }, { name = "loguru", specifier = ">=0.7" }, { name = "playwright", specifier = ">=1.40" }, + { name = "playwright-stealth", specifier = ">=2.0.2" }, { name = "pydantic", specifier = ">=2.0" }, { name = "pydantic-settings", specifier = ">=2.0" }, { name = "python-dotenv", specifier = ">=1.0" }, @@ -431,6 +433,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c8/c4/cc0229fea55c87d6c9c67fe44a21e2cd28d1d558a5478ed4d617e9fb0c93/playwright-1.58.0-py3-none-win_arm64.whl", hash = "sha256:32ffe5c303901a13a0ecab91d1c3f74baf73b84f4bedbb6b935f5bc11cc98e1b", size = 33085919, upload-time = "2026-01-30T15:09:45.71Z" }, ] +[[package]] +name = "playwright-stealth" +version = "2.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "playwright" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/ee/871901103c7b2a12070011fd4d978191f8f962837bf8bb51847274f528fa/playwright_stealth-2.0.2.tar.gz", hash = "sha256:ac57e51873190da5e653e03720e948c8f0a3d06b098f1d56763103d23ee48143", size = 24902, upload-time = "2026-02-13T02:36:25.137Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/30/f95f087f4b071611a7f63a2a0c9af4df3ac046dae2a693bfdacd70512867/playwright_stealth-2.0.2-py3-none-any.whl", hash = "sha256:37a5733f481b9c0ad602cf71491aa5a7c96c2a2fe4fa1e7ab764d2cd35520f2f", size = 33209, upload-time = "2026-02-13T02:36:26.334Z" }, +] + [[package]] name = "pluggy" version = "1.6.0"