Add 8 site adapters, early stop, Telegram notifications, and list command

New site adapters:
- usajobs: multi-keyword sweep (dentist/orthodontics/orthodontist), URL pagination
- docshealth, southernortho: Paylocity platform, keyword in URL
- pdshealth, saltdental: iCIMS Angular platform, URL pagination
- hospitaljobsonline: Load More button, relative URL fix
- aaoinfo: AAO Career Center, click-based AJAX pagination
- aroragroup: Load More + post-scrape keyword filter

Core improvements:
- Early stop pagination: stops when all fingerprints on a page are already in DB
- multi_keyword_mode: separate — one scrape per keyword, cross-sweep dedup
- gimme-job list command to view collected postings
- Stored column in status command
- Telegram notification (replacing KakaoTalk)
- DB repo: count_by_site(), get_existing_fingerprints()

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
main
I Luk Kim 5 months ago
parent 8028d5e566
commit 83fb8542a9

@ -0,0 +1,59 @@
"""AAO Career Center adapter — click-based AJAX pagination."""
from __future__ import annotations
from loguru import logger
from gimme_job.adapters.base import ManifestDrivenAdapter
from gimme_job.adapters.registry import register
from gimme_job.models.dto import JobPostingCandidate, RawJobCard
@register("aaoinfo")
class AAOInfoAdapter(ManifestDrivenAdapter):
_BASE_URL = "https://careers.aaoinfo.org"
def _wait_for_results(self, page) -> None:
try:
page.wait_for_selector("div.job-tile", timeout=15000, state="attached")
except Exception:
logger.debug("[aaoinfo] Results wait timed out — may be zero results")
def collect_cards(self, page, manifest):
self._wait_for_results(page)
return super().collect_cards(page, manifest)
def paginate(self, page, page_index: int, config) -> bool:
"""Click the next page number button in the AJAX pagination."""
if page_index >= config.pagination.max_pages - 1:
return False
next_page = page_index + 2
# Try clicking the specific page number button
next_btn = page.query_selector(f"#page-item-{next_page} a")
if not next_btn:
# Fall back to the last non-active/non-disabled pagination link
next_btn = page.query_selector(
"ul.pagination li.page-item:not(.active):not(.disabled):last-child a"
)
if not next_btn:
logger.debug(f"[aaoinfo] No next page button found — stopping at page {page_index + 1}")
return False
logger.debug(f"[aaoinfo] Paginating to page {next_page}")
next_btn.click()
# Wait for the new page's active indicator to confirm AJAX completed
try:
page.wait_for_selector(f"#page-item-{next_page}.active", timeout=12000)
except Exception:
page.wait_for_timeout(3000)
return True
def normalize(self, raw: RawJobCard) -> JobPostingCandidate:
candidate = super().normalize(raw)
if candidate.job_url and candidate.job_url.startswith("/"):
candidate.job_url = self._BASE_URL + candidate.job_url
return candidate

@ -0,0 +1,68 @@
"""Arora Group Jobs adapter — Load More button, keyword post-filter."""
from __future__ import annotations
from loguru import logger
from gimme_job.adapters.base import ManifestDrivenAdapter
from gimme_job.adapters.registry import register
from gimme_job.models.dto import JobPostingCandidate, RawJobCard
_KEYWORDS = ["dentist", "orthodontic", "orthodontist"]
_MAX_LOAD_MORE = 3 # clicks in addition to the initial load
@register("aroragroup")
class AroraGroupAdapter(ManifestDrivenAdapter):
_BASE_URL = "https://jobs.aroragroup.com"
def collect_cards(self, page, manifest):
"""Load initial results, click 'Load More' up to N times, then filter by keywords."""
try:
page.wait_for_selector(".job-post-row", timeout=15000, state="attached")
except Exception:
logger.debug("[aroragroup] Initial results wait timed out")
for i in range(_MAX_LOAD_MORE):
btn = page.query_selector("button#loadMore")
if not btn or not btn.is_visible():
logger.debug(f"[aroragroup] Load More button gone after {i} extra loads")
break
current_count = len(page.query_selector_all(".job-post-row[onclick]"))
btn.click()
try:
page.wait_for_function(
f"document.querySelectorAll('.job-post-row[onclick]').length > {current_count}",
timeout=15000,
)
logger.debug(f"[aroragroup] Load More {i + 1}/{_MAX_LOAD_MORE}: loaded more cards")
except Exception:
logger.debug(f"[aroragroup] Load More {i + 1}: no new cards appeared — stopping")
break
all_cards = super().collect_cards(page, manifest)
# Filter by keywords in title
filtered = [c for c in all_cards if self._matches_keywords(c.title)]
logger.debug(
f"[aroragroup] Keyword filter: {len(filtered)}/{len(all_cards)} cards matched"
)
return filtered
def paginate(self, page, page_index: int, config) -> bool:
"""Pagination is handled inside collect_cards — always return False."""
return False
def normalize(self, raw: RawJobCard) -> JobPostingCandidate:
candidate = super().normalize(raw)
if candidate.job_url and candidate.job_url.startswith("/"):
candidate.job_url = self._BASE_URL + candidate.job_url
return candidate
@staticmethod
def _matches_keywords(title: str | None) -> bool:
if not title:
return False
title_lower = title.lower()
return any(kw in title_lower for kw in _KEYWORDS)

@ -0,0 +1,34 @@
"""DOCS Health adapter — Paylocity-hosted job board, single-page results."""
from __future__ import annotations
from loguru import logger
from gimme_job.adapters.base import ManifestDrivenAdapter
from gimme_job.adapters.registry import register
from gimme_job.models.dto import JobPostingCandidate, RawJobCard
@register("docshealth")
class DocsHealthAdapter(ManifestDrivenAdapter):
_BASE_URL = "https://recruiting.paylocity.com"
_COMPANY = "DOCS Health"
def collect_cards(self, page, manifest):
# Wait for job listings to render
try:
page.wait_for_selector(".job-listing-job-item", timeout=12000, state="attached")
except Exception:
logger.debug("[docshealth] Results wait timed out — may be zero results")
return super().collect_cards(page, manifest)
def normalize(self, raw: RawJobCard) -> JobPostingCandidate:
candidate = super().normalize(raw)
# Fix relative URL → absolute
if candidate.job_url and candidate.job_url.startswith("/"):
candidate.job_url = self._BASE_URL + candidate.job_url
# Company is always DOCS Health (not in per-card HTML)
candidate.company = self._COMPANY
# Clean posted_text: "03/27/2026 - " → "03/27/2026"
if candidate.posted_text:
candidate.posted_text = candidate.posted_text.replace(" - ", "").strip()
return candidate

@ -0,0 +1,65 @@
"""Hospital Jobs Online adapter — URL-based pagination with page= parameter."""
from __future__ import annotations
import re
from loguru import logger
from gimme_job.adapters.base import ManifestDrivenAdapter
from gimme_job.adapters.registry import register
from gimme_job.models.dto import JobPostingCandidate, RawJobCard
@register("hospitaljobsonline")
class HospitalJobsOnlineAdapter(ManifestDrivenAdapter):
_BASE_URL = "https://www.hospitaljobsonline.com"
def _wait_for_results(self, page) -> None:
try:
page.wait_for_selector("div.jobresult", timeout=15000, state="attached")
except Exception:
logger.debug("[hospitaljobsonline] Results wait timed out — may be zero results")
def collect_cards(self, page, manifest):
self._wait_for_results(page)
return super().collect_cards(page, manifest)
def paginate(self, page, page_index: int, config) -> bool:
"""Navigate to next page by incrementing the ?page= URL parameter."""
if page_index >= config.pagination.max_pages - 1:
return False
current_url = page.url
next_page = page_index + 2
if re.search(r"[?&]page=\d+", current_url):
next_url = re.sub(r"(page=)\d+", f"page={next_page}", current_url)
else:
sep = "&" if "?" in current_url else "?"
next_url = f"{current_url}{sep}page={next_page}"
logger.debug(f"[hospitaljobsonline] Paginating to page {next_page}: {next_url}")
page.goto(next_url, timeout=30000)
self._wait_for_results(page)
return True
def normalize(self, raw: RawJobCard) -> JobPostingCandidate:
candidate = super().normalize(raw)
if candidate.job_url and candidate.job_url.startswith("/"):
candidate.job_url = self._BASE_URL + candidate.job_url
# Parse location text: "1 week ago - Company Name - Location: City, ST"
if candidate.location:
loc = candidate.location
# Extract part after "Location:" if present
m = re.search(r"Location:\s*(.+)$", loc, re.IGNORECASE)
if m:
candidate.location = m.group(1).strip()
else:
# Fallback: take last segment after " - "
parts = [p.strip() for p in loc.split(" - ")]
if len(parts) >= 2:
candidate.location = parts[-1]
return candidate

@ -0,0 +1,48 @@
"""PDS Health adapter — Angular Material job board, URL-based pagination."""
from __future__ import annotations
import re
from loguru import logger
from gimme_job.adapters.base import ManifestDrivenAdapter
from gimme_job.adapters.registry import register
from gimme_job.models.dto import JobPostingCandidate, RawJobCard
@register("pdshealth")
class PdsHealthAdapter(ManifestDrivenAdapter):
def _wait_for_results(self, page) -> None:
try:
page.wait_for_selector(
"mat-expansion-panel.search-result-item",
timeout=15000,
state="attached",
)
except Exception:
logger.debug("[pdshealth] Results wait timed out — may be zero results")
def collect_cards(self, page, manifest):
self._wait_for_results(page)
return super().collect_cards(page, manifest)
def paginate(self, page, page_index: int, config) -> bool:
"""Navigate to next page by incrementing the &page= URL parameter."""
max_pages = config.pagination.max_pages
if page_index >= max_pages - 1:
return False
current_url = page.url
next_page = page_index + 2 # 0-based index → 1-based page param
if re.search(r"[?&]page=\d+", current_url):
next_url = re.sub(r"(page=)\d+", f"page={next_page}", current_url)
else:
sep = "&" if "?" in current_url else "?"
next_url = f"{current_url}{sep}page={next_page}"
logger.debug(f"[pdshealth] Paginating to page {next_page}: {next_url}")
page.goto(next_url, timeout=30000)
self._wait_for_results(page)
return True

@ -0,0 +1,46 @@
"""Salt Dental Partners adapter — iCIMS Angular platform, URL-based pagination."""
from __future__ import annotations
import re
from loguru import logger
from gimme_job.adapters.base import ManifestDrivenAdapter
from gimme_job.adapters.registry import register
@register("saltdental")
class SaltDentalAdapter(ManifestDrivenAdapter):
def _wait_for_results(self, page) -> None:
try:
page.wait_for_selector(
"mat-expansion-panel.search-result-item",
timeout=15000,
state="attached",
)
except Exception:
logger.debug("[saltdental] Results wait timed out — may be zero results")
def collect_cards(self, page, manifest):
self._wait_for_results(page)
return super().collect_cards(page, manifest)
def paginate(self, page, page_index: int, config) -> bool:
"""Navigate to next page by incrementing the &page= URL parameter."""
if page_index >= config.pagination.max_pages - 1:
return False
current_url = page.url
next_page = page_index + 2
if re.search(r"[?&]page=\d+", current_url):
next_url = re.sub(r"(page=)\d+", f"page={next_page}", current_url)
else:
sep = "&" if "?" in current_url else "?"
next_url = f"{current_url}{sep}page={next_page}"
logger.debug(f"[saltdental] Paginating to page {next_page}: {next_url}")
page.goto(next_url, timeout=30000)
self._wait_for_results(page)
return True

@ -0,0 +1,28 @@
"""Southern Orthodontic Partners adapter — Paylocity platform, single page."""
from __future__ import annotations
from loguru import logger
from gimme_job.adapters.base import ManifestDrivenAdapter
from gimme_job.adapters.registry import register
from gimme_job.models.dto import JobPostingCandidate, RawJobCard
@register("southernortho")
class SouthernOrthoAdapter(ManifestDrivenAdapter):
_BASE_URL = "https://recruiting.paylocity.com"
def collect_cards(self, page, manifest):
try:
page.wait_for_selector(".job-listing-job-item", timeout=12000, state="attached")
except Exception:
logger.debug("[southernortho] Results wait timed out — may be zero results")
return super().collect_cards(page, manifest)
def normalize(self, raw: RawJobCard) -> JobPostingCandidate:
candidate = super().normalize(raw)
if candidate.job_url and candidate.job_url.startswith("/"):
candidate.job_url = self._BASE_URL + candidate.job_url
if candidate.posted_text:
candidate.posted_text = candidate.posted_text.replace(" - ", "").strip()
return candidate

@ -0,0 +1,58 @@
"""USAJOBS adapter — waits for JS render, handles relative URLs, URL-based pagination."""
from __future__ import annotations
import re
from loguru import logger
from gimme_job.adapters.base import ManifestDrivenAdapter
from gimme_job.adapters.registry import register
from gimme_job.models.dto import JobPostingCandidate, RawJobCard
@register("usajobs")
class UsaJobsAdapter(ManifestDrivenAdapter):
_BASE_URL = "https://www.usajobs.gov"
def _wait_for_results(self, page) -> None:
"""Wait for JS-rendered results or zero-results heading."""
try:
page.wait_for_selector(
"#search-results .border, h2.font-normal",
timeout=12000,
state="attached",
)
except Exception:
logger.debug("[usajobs] Results wait timed out — may be zero results")
def collect_cards(self, page, manifest):
self._wait_for_results(page)
return super().collect_cards(page, manifest)
def paginate(self, page, page_index: int, config) -> bool:
"""Navigate to next page by incrementing the &p= URL parameter."""
max_pages = config.pagination.max_pages
if page_index >= max_pages - 1:
return False
current_url = page.url
next_page = page_index + 2 # page_index is 0-based; URL param is 1-based
# Replace &p=N or add &p=N
if re.search(r"[?&]p=\d+", current_url):
next_url = re.sub(r"(p=)\d+", f"p={next_page}", current_url)
else:
sep = "&" if "?" in current_url else "?"
next_url = f"{current_url}{sep}p={next_page}"
logger.debug(f"[usajobs] Paginating to page {next_page}: {next_url}")
page.goto(next_url, timeout=30000)
self._wait_for_results(page)
return True
def normalize(self, raw: RawJobCard) -> JobPostingCandidate:
candidate = super().normalize(raw)
# USAJOBS job links are relative: /job/123456 → absolute URL
if candidate.job_url and candidate.job_url.startswith("/"):
candidate.job_url = self._BASE_URL + candidate.job_url
return candidate

@ -94,13 +94,14 @@ def _write_default_global_yaml(path: Path) -> None:
"max_input_items": 200,
},
"notification": {
"provider": "kakaotalk",
"provider": "telegram",
"fallback_markdown": True,
},
}
write_yaml(path, data)
def _check_playwright() -> None:
try:
from playwright.sync_api import sync_playwright
@ -141,6 +142,44 @@ def _check_claude() -> None:
console.print("[yellow]![/yellow] Claude Code CLI not found (needed for learn/repair)")
# ── login ─────────────────────────────────────────────────────────────────────
@app.command()
def login() -> None:
"""Open the JobAgent browser to log in to sites manually.
Browse to any sites (LinkedIn, Indeed, Google, etc.), log in, then press Enter.
All cookies and localStorage are saved in the JobAgent profile and reused on
every subsequent run.
"""
from gimme_job.config import load_global_config
from gimme_job.runtime.browser import BrowserManager
cfg = load_global_config()
console.print(Panel("[bold cyan]gimme-job login[/bold cyan]", expand=False))
console.print(
"Opening JobAgent Chrome profile.\n"
"Log in to any sites you need, then press Enter here to close the browser.\n"
"All cookies and localStorage will be saved and reused on future runs.\n"
)
bm = BrowserManager(
profile_name=cfg.runtime.profile_name,
headless=False,
slow_mo=cfg.runtime.slow_mo_ms,
)
try:
bm.open_context()
bm.new_page()
input(" >> Press Enter when done... ")
finally:
bm.close()
console.print("[green]✓[/green] Browser closed. Session saved to JobAgent profile.")
# ── run ───────────────────────────────────────────────────────────────────────
@ -148,7 +187,7 @@ def _check_claude() -> None:
def run(
site: Optional[str] = typer.Option(None, "--site", help="Run a single site only"),
dry_run: bool = typer.Option(False, "--dry-run", help="Extract but do not save or notify"),
skip_notify: bool = typer.Option(False, "--skip-notify", help="Skip KakaoTalk notification"),
skip_notify: bool = typer.Option(False, "--skip-notify", help="Skip Telegram notification"),
today_only: bool = typer.Option(False, "--today-only", help="Only process today's new items"),
limit_sites: Optional[int] = typer.Option(None, "--limit-sites", help="Max sites to run"),
) -> None:
@ -290,7 +329,7 @@ def test(
def notify(
today: bool = typer.Option(True, "--today/--no-today", help="Re-send today's digest"),
) -> None:
"""Re-send today's job summary via KakaoTalk."""
"""Re-send today's job summary via Telegram."""
from datetime import date
from gimme_job.config import load_global_config
@ -331,6 +370,59 @@ def notify(
console.print("[yellow]![/yellow] Notification failed — saved as markdown fallback")
# ── list ──────────────────────────────────────────────────────────────────────
@app.command(name="list")
def list_postings(
site: Optional[str] = typer.Option(None, "--site", help="Filter by site ID"),
today: bool = typer.Option(False, "--today", help="Only show today's new postings"),
limit: int = typer.Option(50, "--limit", help="Max rows to show"),
) -> None:
"""List collected job postings from the database."""
from datetime import date
from gimme_job.db.engine import get_engine, get_session_factory
from gimme_job.db.repo import JobPostingRepo
from gimme_job.models.db import JobPosting
from sqlalchemy import select
engine = get_engine()
factory = get_session_factory(engine)
table = Table(title="Job Postings", show_header=True, show_lines=False)
table.add_column("#", style="dim", justify="right", width=4)
table.add_column("Site", style="cyan", width=10)
table.add_column("Title", width=40)
table.add_column("Company", width=25)
table.add_column("Location", width=20)
table.add_column("Posted", width=12)
table.add_column("New", width=4)
with factory() as session:
q = select(JobPosting).order_by(JobPosting.first_seen_at.desc())
if site:
q = q.where(JobPosting.site_id == site)
if today:
q = q.where(JobPosting.run_date == date.today(), JobPosting.is_new == True)
q = q.limit(limit)
rows = list(session.execute(q).scalars().all())
for i, p in enumerate(rows, 1):
table.add_row(
str(i),
p.site_id,
p.title[:38] if p.title else "",
(p.company or "")[:23],
(p.location or "")[:18],
p.posted_text or "",
"[green]Y[/green]" if p.is_new else "",
)
console.print(table)
console.print(f"[dim]{len(rows)} rows (limit={limit})[/dim]")
# ── status ────────────────────────────────────────────────────────────────────
@ -344,10 +436,14 @@ def status() -> None:
engine = get_engine()
factory = get_session_factory(engine)
from sqlalchemy import func, select
from gimme_job.models.db import JobPosting
table = Table(title="Site Status", show_header=True)
table.add_column("Site", style="cyan")
table.add_column("Enabled")
table.add_column("Repair?")
table.add_column("Stored", justify="right")
table.add_column("Last Run")
table.add_column("Last Status")
table.add_column("Failures", justify="right")
@ -364,6 +460,9 @@ def status() -> None:
config = session.get(__import__("gimme_job.models.db", fromlist=["SiteConfig"]).SiteConfig, site_id)
recent = run_repo.get_recent_runs(session, site_id, limit=1)
stored = session.execute(
select(func.count()).where(JobPosting.site_id == site_id)
).scalar() or 0
enabled = "[green]yes[/green]" if manifest.enabled else "[dim]no[/dim]"
repair = "[red]YES[/red]" if manifest.repair_needed else "[green]no[/green]"
@ -371,7 +470,7 @@ def status() -> None:
last_status = recent[0].status if recent else "[dim]-[/dim]"
failures = str(config.consecutive_failures) if config else "0"
table.add_row(site_id, enabled, repair, last_run, last_status, failures)
table.add_row(site_id, enabled, repair, str(stored), last_run, last_status, failures)
console.print(table)

@ -40,7 +40,7 @@ class SummarizationConfig(BaseModel):
class NotificationConfig(BaseModel):
provider: str = "kakaotalk"
provider: str = "telegram"
fallback_markdown: bool = True

@ -88,6 +88,24 @@ class JobPostingRepo:
).scalars().all()
)
def count_by_site(self, session: Session, site_id: str) -> int:
"""Count total job postings stored for a site."""
from sqlalchemy import func
return session.execute(
select(func.count()).where(JobPosting.site_id == site_id)
).scalar() or 0
def get_existing_fingerprints(
self, session: Session, fingerprints: list[str]
) -> set[str]:
"""Return the subset of given fingerprints that already exist in the DB."""
if not fingerprints:
return set()
rows = session.execute(
select(JobPosting.fingerprint).where(JobPosting.fingerprint.in_(fingerprints))
).scalars().all()
return set(rows)
class SiteRunRepo:
def record_run(self, session: Session, result: SiteRunResult) -> SiteRun:

@ -70,6 +70,7 @@ class SearchOverride(BaseModel):
date_mode: Optional[str] = None
sort: Optional[str] = None
max_items: Optional[int] = None
multi_keyword_mode: Optional[str] = None # "separate" → one scrape per keyword
class LearnMeta(BaseModel):

@ -10,7 +10,12 @@ from loguru import logger
class BrowserManager:
"""Manages Playwright browser contexts using persistent Chrome profiles."""
def __init__(self, profile_name: str = "JobAgent", headless: bool = True, slow_mo: int = 0):
def __init__(
self,
profile_name: str = "JobAgent",
headless: bool = True,
slow_mo: int = 0,
):
self.profile_name = profile_name
self.headless = headless
self.slow_mo = slow_mo
@ -34,7 +39,6 @@ class BrowserManager:
profile_path = chrome_profile_dir(name)
profile_path.mkdir(parents=True, exist_ok=True)
logger.info(f"Opening browser context: profile={name}, headless={hl}")
self._playwright = sync_playwright().start()
@ -46,12 +50,41 @@ class BrowserManager:
args=[
"--disable-blink-features=AutomationControlled",
"--no-sandbox",
"--disable-infobars",
"--disable-dev-shm-usage",
],
ignore_default_args=["--enable-automation"],
viewport={"width": 1280, "height": 900},
locale="en-US",
user_agent=(
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/134.0.0.0 Safari/537.36"
),
)
# Apply stealth patches to every new page
try:
from playwright_stealth import stealth_sync
self._context.add_init_script("""
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
""")
# stealth_sync works per-page; store reference for use in open_page()
self._stealth_fn = stealth_sync
except ImportError:
self._stealth_fn = None
return self._context
def new_page(self):
"""Open a new page with stealth applied."""
page = self._context.new_page()
if getattr(self, "_stealth_fn", None):
try:
self._stealth_fn(page)
except Exception as e:
logger.debug(f"Stealth apply failed: {e}")
return page
def close(self) -> None:
if self._context:
try:

@ -1,4 +1,4 @@
"""Notification dispatcher: KakaoTalk with Markdown fallback."""
"""Notification dispatcher: Telegram with Markdown fallback."""
from __future__ import annotations
from datetime import date
@ -15,25 +15,23 @@ class NotificationDispatcher:
def send(self, summary: str, run_date: date) -> bool:
"""Send the summary. Returns True if any method succeeded."""
from gimme_job.db.repo import NotificationLogRepo
from gimme_job.runtime.kakao import KakaoTalkClient
from gimme_job.runtime.telegram import TelegramClient
client = KakaoTalkClient()
client = TelegramClient()
success = False
if client.is_configured():
ok = client.send_self_memo(summary)
ok = client.send_message(summary)
if ok:
success = True
self._log_notification(run_date, "kakaotalk", "success")
self._log_notification(run_date, "telegram", "success")
else:
logger.warning("KakaoTalk failed — writing markdown fallback")
self._log_notification(run_date, "kakaotalk", "failed", "send_self_memo returned False")
logger.warning("Telegram failed — writing markdown fallback")
self._log_notification(run_date, "telegram", "failed", "send_message returned False")
else:
logger.warning("KakaoTalk not configured — writing markdown fallback only")
self._log_notification(run_date, "kakaotalk", "skipped", "not configured")
logger.warning("Telegram not configured — writing markdown fallback only")
self._log_notification(run_date, "telegram", "skipped", "not configured")
# Always save markdown fallback if configured
if self.cfg.notification.fallback_markdown:
path = self.send_markdown_fallback(summary, run_date)
logger.info(f"Markdown saved: {path}")

@ -56,9 +56,10 @@ class RunOrchestrator:
# Record to DB (unless dry run)
if not dry_run:
with self.session_factory() as session:
from gimme_job.db.repo import SiteRunRepo
from gimme_job.runtime.health import update_site_status
from gimme_job.db.engine import db_session
from gimme_job.db.repo import SiteRunRepo
from gimme_job.runtime.health import update_site_status
with db_session() as session:
SiteRunRepo().record_run(session, result)
update_site_status(site_id, result, session)
@ -97,7 +98,21 @@ class RunOrchestrator:
from gimme_job.utils.paths import screenshot_path, dom_snapshot_path, trace_path
try:
candidates = self._scrape_site(site_id, manifest, query, run_id, result)
if manifest.search_override.multi_keyword_mode == "separate" and query.keywords:
# Run one scrape per keyword and merge results
from gimme_job.runtime.dedupe import deduplicate_in_batch
all_candidates = []
for kw in query.keywords:
from gimme_job.models.dto import SearchQuery
kw_query = query.model_copy(update={"keywords": [kw]})
kw_run_id = f"{run_id}-{kw[:8]}"
logger.info(f"[{site_id}] Keyword sweep: '{kw}'")
kw_candidates = self._scrape_site(site_id, manifest, kw_query, kw_run_id, result, dry_run=dry_run)
all_candidates.extend(kw_candidates)
# Deduplicate across keyword sweeps by fingerprint
candidates = deduplicate_in_batch(all_candidates)
else:
candidates = self._scrape_site(site_id, manifest, query, run_id, result, dry_run=dry_run)
except Exception as e:
logger.error(f"[{site_id}] Scrape error: {e}")
result.status = RunStatus.FAILED
@ -108,8 +123,9 @@ class RunOrchestrator:
result.items_found = len(candidates)
if not dry_run and candidates:
with self.session_factory() as session:
from gimme_job.db.repo import JobPostingRepo
from gimme_job.db.engine import db_session
from gimme_job.db.repo import JobPostingRepo
with db_session() as session:
total, new_count = JobPostingRepo().upsert_candidates(session, candidates, run_date)
result.new_items = new_count
elif dry_run:
@ -131,6 +147,7 @@ class RunOrchestrator:
query: SearchQuery,
run_id: str,
result: SiteRunResult,
dry_run: bool = False,
):
from gimme_job.adapters.registry import get_adapter
from gimme_job.runtime.browser import BrowserManager
@ -141,13 +158,17 @@ class RunOrchestrator:
headless = manifest.browser.headless_on_run
profile = manifest.browser.profile_name or self.cfg.runtime.profile_name
bm = BrowserManager(profile_name=profile, headless=headless, slow_mo=self.cfg.runtime.slow_mo_ms)
bm = BrowserManager(
profile_name=profile,
headless=headless,
slow_mo=self.cfg.runtime.slow_mo_ms,
)
all_cards = []
try:
context = bm.open_context()
bm.start_tracing()
page = context.new_page()
page = bm.new_page()
# Set timeouts
page.set_default_timeout(self.cfg.runtime.default_timeout_ms)
@ -177,6 +198,17 @@ class RunOrchestrator:
except Exception:
logger.warning(f"[{site_id}] Wait selector timed out — continuing anyway")
# Check if site has existing data (for early stop on update runs)
site_has_data = False
if not dry_run:
try:
from gimme_job.db.engine import db_session
from gimme_job.db.repo import JobPostingRepo
with db_session() as _s:
site_has_data = JobPostingRepo().count_by_site(_s, site_id) > 0
except Exception:
pass
# Collect cards across pages
max_pages = min(
manifest.pagination.max_pages,
@ -187,6 +219,35 @@ class RunOrchestrator:
all_cards.extend(page_cards)
logger.debug(f"[{site_id}] Page {page_idx+1}: {len(page_cards)} cards")
if not page_cards:
break # no results on this page — stop paginating
# Early stop: if site has existing data and this page's cards are all known,
# we've caught up to previously stored content — no need to paginate further.
if site_has_data and page_idx >= 1:
try:
from gimme_job.db.engine import db_session
from gimme_job.db.repo import JobPostingRepo
page_fps = []
for card in page_cards:
try:
c = adapter.normalize(card)
if c.fingerprint:
page_fps.append(c.fingerprint)
except Exception:
pass
if page_fps:
with db_session() as _s:
known = JobPostingRepo().get_existing_fingerprints(_s, page_fps)
if len(known) >= len(page_fps):
logger.info(
f"[{site_id}] Page {page_idx+1}: all {len(page_fps)} items"
" already in DB — stopping early"
)
break
except Exception as e:
logger.debug(f"[{site_id}] Early stop check failed: {e}")
if page_idx < max_pages - 1:
if not adapter.paginate(page, page_idx, manifest):
break
@ -247,22 +308,25 @@ class RunOrchestrator:
def _summarize_and_notify(self, summary: RunSummary, run_date: date) -> None:
try:
with self.session_factory() as session:
from gimme_job.db.repo import JobPostingRepo, SummaryRepo
from gimme_job.db.engine import db_session
from gimme_job.db.repo import JobPostingRepo, SummaryRepo
with db_session() as session:
postings = JobPostingRepo().get_today_new(session, run_date)
if not postings:
return
if not postings:
return
from gimme_job.runtime.summarizer import OllamaSummarizer
summarizer = OllamaSummarizer(
base_url=self.cfg.summarization.ollama_base_url,
model=self.cfg.summarization.model,
temperature=self.cfg.summarization.temperature,
)
text = summarizer.summarize(postings)
summary.summary_text = text
from gimme_job.runtime.summarizer import OllamaSummarizer
summarizer = OllamaSummarizer(
base_url=self.cfg.summarization.ollama_base_url,
model=self.cfg.summarization.model,
temperature=self.cfg.summarization.temperature,
)
text = summarizer.summarize(postings)
summary.summary_text = text
with db_session() as session:
SummaryRepo().save_summary(session, run_date, text, self.cfg.summarization.model)
from gimme_job.runtime.notifier import NotificationDispatcher

@ -0,0 +1,53 @@
"""Telegram notification client — sends messages via Bot API."""
from __future__ import annotations
import os
from typing import Optional
from loguru import logger
TELEGRAM_API_URL = "https://api.telegram.org/bot{token}/sendMessage"
MAX_TEXT_LENGTH = 4096 # Telegram message limit
class TelegramClient:
def __init__(
self,
bot_token: Optional[str] = None,
chat_id: Optional[str] = None,
):
self.bot_token = bot_token or os.environ.get("TELEGRAM_BOT_TOKEN", "")
self.chat_id = chat_id or os.environ.get("TELEGRAM_CHAT_ID", "")
def is_configured(self) -> bool:
return bool(self.bot_token and self.chat_id)
def send_message(self, text: str) -> bool:
"""Send a message to the configured chat. Returns True on success."""
if not self.is_configured():
logger.warning("Telegram not configured (missing BOT_TOKEN or CHAT_ID)")
return False
if len(text) > MAX_TEXT_LENGTH:
text = text[: MAX_TEXT_LENGTH - 20] + "\n\n[...truncated]"
try:
import httpx
response = httpx.post(
TELEGRAM_API_URL.format(token=self.bot_token),
json={
"chat_id": self.chat_id,
"text": text,
"parse_mode": "HTML",
},
timeout=15.0,
)
if response.status_code == 200 and response.json().get("ok"):
logger.info("Telegram message sent successfully")
return True
else:
logger.error(f"Telegram API error: {response.text[:200]}")
return False
except Exception as e:
logger.error(f"Telegram send failed: {e}")
return False

@ -17,6 +17,7 @@ dependencies = [
"jinja2>=3.1",
"pyyaml>=6.0",
"python-dotenv>=1.0",
"playwright-stealth>=2.0.2",
]
[project.scripts]

@ -0,0 +1,60 @@
site_id: aaoinfo
enabled: true
repair_needed: false
identity:
label: AAO Career Center
category: aggregator
base_url: https://careers.aaoinfo.org
start_url_template: "https://careers.aaoinfo.org/jobs/"
browser:
profile_mode: persistent_chrome_profile
profile_name: JobAgent
headed_on_learn: true
headless_on_run: true
search:
keyword_mode: none
location_mode: none
sort_mode: none
date_mode: none
result_list_wait_selector: "div.job-tile"
pagination:
mode: none
max_pages: 10
extract:
container_selectors:
- "div.job-tile:not(.job-mo)"
fields:
title:
text:
- ".job-title a"
company:
text:
- ".job-company-row"
location:
text:
- ".job-location"
posted_text:
text:
- ".job-posted-date"
url:
attr:
selector: ".job-title a"
name: href
post_filters:
include_posted_text: []
healthcheck:
min_items_expected: 0
required_fields:
- title
fail_if_zero_when_known_active: false
learn:
last_learned_at: null
source_url: null

@ -0,0 +1,56 @@
site_id: aroragroup
enabled: true
repair_needed: false
identity:
label: Arora Group Jobs
category: employer
base_url: https://jobs.aroragroup.com
start_url_template: "https://jobs.aroragroup.com/"
browser:
profile_mode: persistent_chrome_profile
profile_name: JobAgent
headed_on_learn: true
headless_on_run: true
search:
keyword_mode: none
location_mode: none
sort_mode: none
date_mode: none
result_list_wait_selector: ".job-post-row"
pagination:
mode: none
max_pages: 1
extract:
container_selectors:
- ".job-post-row[onclick]"
fields:
title:
text:
- ".POST_TITLE"
- "h3.job-post-title a"
location:
text:
- ".POST_LOCATION"
- ".job-post-location"
url:
attr:
selector: "a.job-post-href"
name: href
post_filters:
include_posted_text: []
healthcheck:
min_items_expected: 0
required_fields:
- title
fail_if_zero_when_known_active: false
learn:
last_learned_at: null
source_url: null

@ -0,0 +1,64 @@
site_id: docshealth
enabled: true
repair_needed: false
identity:
label: DOCS Health
category: employer
base_url: https://recruiting.paylocity.com
start_url_template: "https://recruiting.paylocity.com/recruiting/jobs/All/4f5b87cb-fda9-4243-976b-499bbf0ce9fc/DOCS-Health?search={keywords_urlencoded}&location=All%20Locations&department=All%20Departments"
browser:
profile_mode: persistent_chrome_profile
profile_name: JobAgent
headed_on_learn: true
headless_on_run: true
search:
keyword_mode: url_param
location_mode: url_param
sort_mode: none
date_mode: none
result_list_wait_selector: ".job-listing-job-item"
search_override:
keywords:
- dentist
- orthodontics
- orthodontist
multi_keyword_mode: separate
pagination:
mode: none
max_pages: 1
extract:
container_selectors:
- "div.job-listing-job-item"
fields:
title:
text:
- ".job-item-title a"
location:
text:
- ".location-column .job-item-normal"
posted_text:
text:
- ".job-title-column > span:not(.job-item-title):not(.job-item-normal)"
url:
attr:
selector: ".job-item-title a"
name: href
post_filters:
include_posted_text: []
healthcheck:
min_items_expected: 0
required_fields:
- title
fail_if_zero_when_known_active: false
learn:
last_learned_at: null
source_url: null

@ -5,7 +5,7 @@ runtime:
slow_mo_ms: 0
default_timeout_ms: 15000
navigation_timeout_ms: 30000
max_pages_per_site: 3
max_pages_per_site: 10
min_delay_ms: 1200
max_delay_ms: 3500

@ -0,0 +1,60 @@
site_id: hospitaljobsonline
enabled: true
repair_needed: false
identity:
label: Hospital Jobs Online
category: aggregator
base_url: https://www.hospitaljobsonline.com
start_url_template: "https://www.hospitaljobsonline.com/jobs/q-korea-jobs//?page=1&limit=100"
browser:
profile_mode: persistent_chrome_profile
profile_name: JobAgent
headed_on_learn: true
headless_on_run: true
search:
keyword_mode: none
location_mode: none
sort_mode: none
date_mode: none
result_list_wait_selector: "div.jobresult"
pagination:
mode: url_increment
max_pages: 10
extract:
container_selectors:
- "div.jobresult"
fields:
title:
text:
- ".jobtitle a"
company:
text:
- ".location a"
location:
text:
- ".location"
posted_text:
text:
- ".location .date"
url:
attr:
selector: ".jobtitle a"
name: href
post_filters:
include_posted_text: []
healthcheck:
min_items_expected: 0
required_fields:
- title
fail_if_zero_when_known_active: false
learn:
last_learned_at: null
source_url: null

@ -1,5 +1,5 @@
site_id: indeed
enabled: true
enabled: false
repair_needed: false
identity:
@ -12,7 +12,7 @@ browser:
profile_mode: persistent_chrome_profile
profile_name: JobAgent
headed_on_learn: true
headless_on_run: true
headless_on_run: false
search:
keyword_mode: url_param

@ -0,0 +1,74 @@
site_id: linkedin
enabled: true
repair_needed: false
identity:
label: LinkedIn Jobs
category: aggregator
base_url: https://www.linkedin.com
start_url_template: "https://www.linkedin.com/jobs/search/?keywords={keywords_urlencoded}&f_TPR=r86400&sortBy=DD"
browser:
profile_mode: persistent_chrome_profile
profile_name: JobAgent
headed_on_learn: true
headless_on_run: false
search:
keyword_mode: url_param
location_mode: url_param
sort_mode: url_param
date_mode: url_param
result_list_wait_selector: ".jobs-search__results-list li, .scaffold-layout__list-container li"
pagination:
mode: next_button
next_button_selectors:
- "button[aria-label='View next page']"
- "button[aria-label*='next']"
- "li.artdeco-pagination__indicator--number.selected + li button"
max_pages: 3
extract:
container_selectors:
- ".jobs-search__results-list li"
- ".scaffold-layout__list-container li"
- "li:has(.base-card)"
fields:
title:
text:
- "h3.base-search-card__title"
- ".job-card-list__title"
- "h3"
- "a[data-control-name='job_card_title']"
company:
text:
- "h4.base-search-card__subtitle"
- ".job-card-container__primary-description"
- "a.job-card-container__company-name"
location:
text:
- "span.job-search-card__location"
- ".job-card-container__metadata-item"
posted_text:
text:
- "time"
- ".job-search-card__listdate"
- "span[class*='listdate']"
url:
attr:
selector: "a.base-card__full-link, a[data-control-name='job_card_title'], a[href*='/jobs/view/']"
name: href
post_filters:
include_posted_text: []
healthcheck:
min_items_expected: 0
required_fields:
- title
fail_if_zero_when_known_active: false
learn:
last_learned_at: null
source_url: null

@ -0,0 +1,57 @@
site_id: pdshealth
enabled: true
repair_needed: false
identity:
label: PDS Health
category: employer
base_url: https://jobs.pdshealth.com
start_url_template: "https://jobs.pdshealth.com/jobs?keywords=orthodontist&sortBy=relevance&page=1&limit=100"
browser:
profile_mode: persistent_chrome_profile
profile_name: JobAgent
headed_on_learn: true
headless_on_run: true
search:
keyword_mode: none
location_mode: none
sort_mode: none
date_mode: none
result_list_wait_selector: "mat-expansion-panel.search-result-item"
pagination:
mode: url_increment
max_pages: 20
extract:
container_selectors:
- "mat-expansion-panel.search-result-item"
fields:
title:
text:
- "a.job-title-link"
company:
text:
- ".label-value.tags1"
location:
text:
- ".label-value.location"
url:
attr:
selector: "a.job-title-link"
name: href
post_filters:
include_posted_text: []
healthcheck:
min_items_expected: 0
required_fields:
- title
fail_if_zero_when_known_active: false
learn:
last_learned_at: null
source_url: null

@ -0,0 +1,57 @@
site_id: saltdental
enabled: true
repair_needed: false
identity:
label: Salt Dental Partners
category: employer
base_url: https://jobs.saltdentalpartners.com
start_url_template: "https://jobs.saltdentalpartners.com/jobs?keywords=orthodontist&sortBy=relevance&page=1"
browser:
profile_mode: persistent_chrome_profile
profile_name: JobAgent
headed_on_learn: true
headless_on_run: true
search:
keyword_mode: none
location_mode: none
sort_mode: none
date_mode: none
result_list_wait_selector: "mat-expansion-panel.search-result-item"
pagination:
mode: url_increment
max_pages: 5
extract:
container_selectors:
- "mat-expansion-panel.search-result-item"
fields:
title:
text:
- "a.job-title-link"
company:
text:
- ".label-value.tags1"
location:
text:
- ".label-value.location"
url:
attr:
selector: "a.job-title-link"
name: href
post_filters:
include_posted_text: []
healthcheck:
min_items_expected: 0
required_fields:
- title
fail_if_zero_when_known_active: false
learn:
last_learned_at: null
source_url: null

@ -0,0 +1,57 @@
site_id: southernortho
enabled: true
repair_needed: false
identity:
label: Southern Orthodontic Partners
category: employer
base_url: https://recruiting.paylocity.com
start_url_template: "https://recruiting.paylocity.com/recruiting/jobs/All/06144481-7908-457c-9312-51cf7b58cf01/Southern-Orthodontic-Partners-Mgmt-LLC?search=orthodontist&location=All%20Locations&department=All%20Departments"
browser:
profile_mode: persistent_chrome_profile
profile_name: JobAgent
headed_on_learn: true
headless_on_run: true
search:
keyword_mode: none
location_mode: none
sort_mode: none
date_mode: none
result_list_wait_selector: ".job-listing-job-item"
pagination:
mode: none
max_pages: 1
extract:
container_selectors:
- "div.job-listing-job-item"
fields:
title:
text:
- ".job-item-title a"
company:
text:
- ".location-column .job-item-normal"
posted_text:
text:
- ".job-title-column > span:not(.job-item-title):not(.job-item-normal)"
url:
attr:
selector: ".job-item-title a"
name: href
post_filters:
include_posted_text: []
healthcheck:
min_items_expected: 0
required_fields:
- title
fail_if_zero_when_known_active: false
learn:
last_learned_at: null
source_url: null

@ -0,0 +1,70 @@
site_id: usajobs
enabled: true
repair_needed: false
identity:
label: USAJOBS
category: government
base_url: https://www.usajobs.gov
start_url_template: "https://www.usajobs.gov/Search/Results?k={keywords_urlencoded}&p=1"
browser:
profile_mode: persistent_chrome_profile
profile_name: JobAgent
headed_on_learn: true
headless_on_run: true
search:
keyword_mode: url_param
location_mode: url_param
sort_mode: url_param
date_mode: url_param
result_list_wait_selector: ""
pagination:
mode: next_button
next_button_selectors:
- "button#page-m-next"
- "button[title='Go to next page']"
max_pages: 10
extract:
container_selectors:
- "#search-results .border.border-gray-lighter"
fields:
title:
text:
- "h2 a"
company:
text:
- "p strong"
location:
text:
- "div:nth-child(2) div:first-child p:nth-child(3)"
posted_text:
text:
- ".italic"
url:
attr:
selector: "h2 a"
name: href
search_override:
keywords:
- dentist
- orthodontics
- orthodontist
multi_keyword_mode: separate
post_filters:
include_posted_text: []
healthcheck:
min_items_expected: 0
required_fields:
- title
fail_if_zero_when_known_active: false
learn:
last_learned_at: null
source_url: null

@ -156,6 +156,7 @@ dependencies = [
{ name = "jinja2" },
{ name = "loguru" },
{ name = "playwright" },
{ name = "playwright-stealth" },
{ name = "pydantic" },
{ name = "pydantic-settings" },
{ name = "python-dotenv" },
@ -179,6 +180,7 @@ requires-dist = [
{ name = "jinja2", specifier = ">=3.1" },
{ name = "loguru", specifier = ">=0.7" },
{ name = "playwright", specifier = ">=1.40" },
{ name = "playwright-stealth", specifier = ">=2.0.2" },
{ name = "pydantic", specifier = ">=2.0" },
{ name = "pydantic-settings", specifier = ">=2.0" },
{ name = "python-dotenv", specifier = ">=1.0" },
@ -431,6 +433,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/c8/c4/cc0229fea55c87d6c9c67fe44a21e2cd28d1d558a5478ed4d617e9fb0c93/playwright-1.58.0-py3-none-win_arm64.whl", hash = "sha256:32ffe5c303901a13a0ecab91d1c3f74baf73b84f4bedbb6b935f5bc11cc98e1b", size = 33085919, upload-time = "2026-01-30T15:09:45.71Z" },
]
[[package]]
name = "playwright-stealth"
version = "2.0.2"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "playwright" },
]
sdist = { url = "https://files.pythonhosted.org/packages/61/ee/871901103c7b2a12070011fd4d978191f8f962837bf8bb51847274f528fa/playwright_stealth-2.0.2.tar.gz", hash = "sha256:ac57e51873190da5e653e03720e948c8f0a3d06b098f1d56763103d23ee48143", size = 24902, upload-time = "2026-02-13T02:36:25.137Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/f1/30/f95f087f4b071611a7f63a2a0c9af4df3ac046dae2a693bfdacd70512867/playwright_stealth-2.0.2-py3-none-any.whl", hash = "sha256:37a5733f481b9c0ad602cf71491aa5a7c96c2a2fe4fa1e7ab764d2cd35520f2f", size = 33209, upload-time = "2026-02-13T02:36:26.334Z" },
]
[[package]]
name = "pluggy"
version = "1.6.0"

Loading…
Cancel
Save