From 89ca7a400661865b519142292ad627481d2f1816 Mon Sep 17 00:00:00 2001 From: I Luk Kim Date: Sat, 28 Mar 2026 02:34:18 -0700 Subject: [PATCH] Fix warnings, LinkedIn selector update, Telegram notify listing, usajobs title filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - LinkedIn: new URL with geoId, updated selectors to li[data-occludable-job-id], scroll adapter, pagination disabled - usajobs: add include_title_keywords post-filter (dentist/orthodontist/orthodontic) - aroragroup: fix AJAX load via doloadJBSearchList(), fix container selector - govtjobs/srpmic: clear result_list_wait_selector to avoid 15s timeout on 0-result pages - orchestrator: wait selector timeout WARNING → DEBUG - extractor: no-container WARNING → DEBUG - notifier: provider-based dispatch (telegram/kakaotalk), build_listing_messages() for Telegram - cli notify: send job listing with links instead of Ollama summary - global.yaml: provider set to telegram Co-Authored-By: Claude Sonnet 4.6 --- gimme_job/adapters/aroragroup.py | 18 +++-- gimme_job/adapters/linkedin.py | 34 ++++++++++ gimme_job/cli.py | 58 +++++++++-------- gimme_job/models/manifest.py | 1 + gimme_job/runtime/extractor.py | 2 +- gimme_job/runtime/notifier.py | 105 ++++++++++++++++++++++++++---- gimme_job/runtime/orchestrator.py | 16 ++++- sites/aroragroup.yaml | 2 +- sites/global.yaml | 2 +- sites/govtjobs.yaml | 2 +- sites/linkedin.yaml | 35 +++------- sites/srpmic.yaml | 2 +- sites/usajobs.yaml | 4 ++ 13 files changed, 202 insertions(+), 79 deletions(-) create mode 100644 gimme_job/adapters/linkedin.py diff --git a/gimme_job/adapters/aroragroup.py b/gimme_job/adapters/aroragroup.py index 12d59a9..56c79e0 100644 --- a/gimme_job/adapters/aroragroup.py +++ b/gimme_job/adapters/aroragroup.py @@ -16,11 +16,19 @@ class AroraGroupAdapter(ManifestDrivenAdapter): _BASE_URL = "https://jobs.aroragroup.com" def collect_cards(self, page, manifest): - """Load initial results, click 'Load More' up to N times, then filter by keywords.""" + """Trigger AJAX job load, click 'Load More' up to N times, then filter by keywords.""" + # Trigger initial job list load via the page's JS function try: - page.wait_for_selector(".job-post-row", timeout=15000, state="attached") + page.evaluate("doloadJBSearchList(1)") except Exception: - logger.debug("[aroragroup] Initial results wait timed out") + pass + + # Wait for at least one title to be populated + try: + page.wait_for_selector(".POST_TITLE:not(:empty)", timeout=15000, state="attached") + except Exception: + logger.debug("[aroragroup] Job titles did not populate — no results or load failed") + return [] for i in range(_MAX_LOAD_MORE): btn = page.query_selector("button#loadMore") @@ -28,12 +36,12 @@ class AroraGroupAdapter(ManifestDrivenAdapter): logger.debug(f"[aroragroup] Load More button gone after {i} extra loads") break - current_count = len(page.query_selector_all(".job-post-row[onclick]")) + current_count = len(page.query_selector_all(".POST_TITLE:not(:empty)")) btn.click() try: page.wait_for_function( - f"document.querySelectorAll('.job-post-row[onclick]').length > {current_count}", + f"document.querySelectorAll('.POST_TITLE:not(:empty)').length > {current_count}", timeout=15000, ) logger.debug(f"[aroragroup] Load More {i + 1}/{_MAX_LOAD_MORE}: loaded more cards") diff --git a/gimme_job/adapters/linkedin.py b/gimme_job/adapters/linkedin.py new file mode 100644 index 0000000..13216a4 --- /dev/null +++ b/gimme_job/adapters/linkedin.py @@ -0,0 +1,34 @@ +"""LinkedIn Jobs adapter — scroll to load all cards on first page only.""" +from __future__ import annotations + +from loguru import logger + +from gimme_job.adapters.base import ManifestDrivenAdapter +from gimme_job.adapters.registry import register +from gimme_job.models.dto import JobPostingCandidate, RawJobCard + + +@register("linkedin") +class LinkedInAdapter(ManifestDrivenAdapter): + + def collect_cards(self, page, manifest) -> list[RawJobCard]: + """Scroll each card into view to trigger lazy-loading, then extract.""" + self._scroll_to_load(page) + return super().collect_cards(page, manifest) + + def normalize(self, raw: RawJobCard) -> JobPostingCandidate: + return super().normalize(raw) + + @staticmethod + def _scroll_to_load(page) -> None: + """Scroll each job card into view to force LinkedIn's virtual list to render.""" + try: + page.evaluate("""() => { + const cards = Array.from(document.querySelectorAll('li[data-occludable-job-id]')); + cards.forEach(card => card.scrollIntoView({ behavior: 'instant', block: 'center' })); + }""") + page.wait_for_timeout(800) + page.evaluate("window.scrollTo(0, 0)") + page.wait_for_timeout(400) + except Exception as e: + logger.debug(f"[linkedin] scroll_to_load failed: {e}") diff --git a/gimme_job/cli.py b/gimme_job/cli.py index a6e584f..e30d09c 100644 --- a/gimme_job/cli.py +++ b/gimme_job/cli.py @@ -5,6 +5,9 @@ import sys from pathlib import Path from typing import Optional +import dotenv +dotenv.load_dotenv() + import typer from rich.console import Console from rich.panel import Panel @@ -327,15 +330,16 @@ def test( @app.command() def notify( - today: bool = typer.Option(True, "--today/--no-today", help="Re-send today's digest"), + today: bool = typer.Option(True, "--today/--no-today", help="Re-send today's job listing"), ) -> None: - """Re-send today's job summary via Telegram.""" + """Send today's new job listings via Telegram (one message per site group).""" from datetime import date from gimme_job.config import load_global_config from gimme_job.db.engine import get_engine, get_session_factory - from gimme_job.db.repo import JobPostingRepo, SummaryRepo - from gimme_job.runtime.notifier import NotificationDispatcher + from gimme_job.db.repo import JobPostingRepo + from gimme_job.runtime.notifier import NotificationDispatcher, build_listing_messages + from gimme_job.runtime.telegram import TelegramClient cfg = load_global_config() engine = get_engine() @@ -343,31 +347,33 @@ def notify( run_date = date.today() with factory() as session: - summary_repo = SummaryRepo() - summary = summary_repo.get_latest(session, run_date) - - if summary: - text = summary.content + postings = JobPostingRepo().get_today_new(session, run_date) + + if not postings: + console.print("[yellow]No new postings found for today.[/yellow]") + raise typer.Exit(0) + + chunks = build_listing_messages(postings) + console.print(f"Sending {len(chunks)} message(s) for {len(postings)} postings...") + + client = TelegramClient() + sent = 0 + if client.is_configured(): + for chunk in chunks: + if client.send_message(chunk): + sent += 1 + if sent == len(chunks): + console.print(f"[green]✓[/green] All {sent} message(s) sent") else: - # Build summary from today's postings - postings = JobPostingRepo().get_today_new(session, run_date) - if not postings: - console.print("[yellow]No new postings found for today.[/yellow]") - raise typer.Exit(0) - from gimme_job.runtime.summarizer import OllamaSummarizer - summarizer = OllamaSummarizer( - base_url=cfg.summarization.ollama_base_url, - model=cfg.summarization.model, - temperature=cfg.summarization.temperature, - ) - text = summarizer.summarize(postings) + console.print(f"[yellow]![/yellow] Sent {sent}/{len(chunks)} messages") + else: + console.print("[yellow]Telegram not configured[/yellow]") + # Always write markdown fallback dispatcher = NotificationDispatcher(global_config=cfg, session_factory=factory) - ok = dispatcher.send(text, run_date) - if ok: - console.print("[green]✓[/green] Notification sent") - else: - console.print("[yellow]![/yellow] Notification failed — saved as markdown fallback") + full_text = "\n\n".join(chunks) + path = dispatcher.send_markdown_fallback(full_text, run_date) + console.print(f"Markdown saved: {path}") # ── list ────────────────────────────────────────────────────────────────────── diff --git a/gimme_job/models/manifest.py b/gimme_job/models/manifest.py index 43daabb..e1f6036 100644 --- a/gimme_job/models/manifest.py +++ b/gimme_job/models/manifest.py @@ -53,6 +53,7 @@ class ExtractConfig(BaseModel): class PostFilterConfig(BaseModel): include_posted_text: list[str] = Field(default_factory=list) + include_title_keywords: list[str] = Field(default_factory=list) class HealthcheckConfig(BaseModel): diff --git a/gimme_job/runtime/extractor.py b/gimme_job/runtime/extractor.py index dfce29f..2748fe1 100644 --- a/gimme_job/runtime/extractor.py +++ b/gimme_job/runtime/extractor.py @@ -60,7 +60,7 @@ def extract_cards_from_page(page: "Page", extract_config: ExtractConfig) -> list continue if not container_elements: - logger.warning("No container elements found with any selector") + logger.debug("No container elements found with any selector") return cards for element in container_elements: diff --git a/gimme_job/runtime/notifier.py b/gimme_job/runtime/notifier.py index 4d753b6..5cfccdf 100644 --- a/gimme_job/runtime/notifier.py +++ b/gimme_job/runtime/notifier.py @@ -1,12 +1,73 @@ -"""Notification dispatcher: Telegram with Markdown fallback.""" +"""Notification dispatcher: Telegram or KakaoTalk with Markdown fallback.""" from __future__ import annotations from datetime import date +from typing import TYPE_CHECKING from loguru import logger from gimme_job.config import GlobalConfig +if TYPE_CHECKING: + from gimme_job.models.db import JobPosting + +_TELEGRAM_MAX_LEN = 4000 # leave room for safety margin + + +def build_listing_messages(postings: list["JobPosting"]) -> list[str]: + """Format job postings as Telegram HTML chunks (≤4000 chars each).""" + from collections import defaultdict + + by_site: dict[str, list] = defaultdict(list) + for p in postings: + by_site[p.site_id].append(p) + + total = len(postings) + header = f"📋 오늘의 신규 채용 공고 ({total}개)\n" + + chunks: list[str] = [] + current = header + + for site_id, site_postings in sorted(by_site.items()): + site_block = f"\n── {site_id.upper()} ({len(site_postings)}) ──\n" + for p in site_postings: + # Title line + title = _escape_html(p.title) + line = f"• {title}" + + # Details line + details = [] + if p.company: + details.append(_escape_html(p.company)) + if p.location: + details.append(_escape_html(p.location)) + if p.employment_type: + details.append(_escape_html(p.employment_type)) + if details: + line += f"\n {' | '.join(details)}" + + # Link + if p.job_url: + line += f'\n 링크' + + site_block += line + "\n" + + # Flush chunk if adding this site would exceed limit + if len(current) + len(site_block) > _TELEGRAM_MAX_LEN: + chunks.append(current.rstrip()) + current = site_block + else: + current += site_block + + if current.strip(): + chunks.append(current.rstrip()) + + return chunks or ["신규 채용 공고가 없습니다."] + + +def _escape_html(text: str) -> str: + return text.replace("&", "&").replace("<", "<").replace(">", ">") + class NotificationDispatcher: def __init__(self, global_config: GlobalConfig, session_factory): @@ -15,22 +76,38 @@ class NotificationDispatcher: def send(self, summary: str, run_date: date) -> bool: """Send the summary. Returns True if any method succeeded.""" - from gimme_job.runtime.telegram import TelegramClient - - client = TelegramClient() + provider = self.cfg.notification.provider success = False - if client.is_configured(): - ok = client.send_message(summary) - if ok: - success = True - self._log_notification(run_date, "telegram", "success") + if provider == "kakaotalk": + from gimme_job.runtime.kakao import KakaoTalkClient + client = KakaoTalkClient() + if client.is_configured(): + ok = client.send_self_memo(summary) + if ok: + success = True + self._log_notification(run_date, "kakaotalk", "success") + else: + logger.warning("KakaoTalk failed — writing markdown fallback") + self._log_notification(run_date, "kakaotalk", "failed", "send_self_memo returned False") else: - logger.warning("Telegram failed — writing markdown fallback") - self._log_notification(run_date, "telegram", "failed", "send_message returned False") - else: - logger.warning("Telegram not configured — writing markdown fallback only") - self._log_notification(run_date, "telegram", "skipped", "not configured") + logger.warning("KakaoTalk not configured — writing markdown fallback only") + self._log_notification(run_date, "kakaotalk", "skipped", "not configured") + + else: # telegram (default) + from gimme_job.runtime.telegram import TelegramClient + client = TelegramClient() + if client.is_configured(): + ok = client.send_message(summary) + if ok: + success = True + self._log_notification(run_date, "telegram", "success") + else: + logger.warning("Telegram failed — writing markdown fallback") + self._log_notification(run_date, "telegram", "failed", "send_message returned False") + else: + logger.warning("Telegram not configured — writing markdown fallback only") + self._log_notification(run_date, "telegram", "skipped", "not configured") if self.cfg.notification.fallback_markdown: path = self.send_markdown_fallback(summary, run_date) diff --git a/gimme_job/runtime/orchestrator.py b/gimme_job/runtime/orchestrator.py index b3addab..c8a0f27 100644 --- a/gimme_job/runtime/orchestrator.py +++ b/gimme_job/runtime/orchestrator.py @@ -196,7 +196,7 @@ class RunOrchestrator: state="attached", ) except Exception: - logger.warning(f"[{site_id}] Wait selector timed out — continuing anyway") + logger.debug(f"[{site_id}] Wait selector timed out — continuing anyway") # Check if site has existing data (for early stop on update runs) site_has_data = False @@ -292,14 +292,24 @@ class RunOrchestrator: # Apply post-filters if manifest.post_filters.include_posted_text: - from gimme_job.models.dto import RawJobCard - # Re-filter candidates based on posted_text include_lower = [t.lower() for t in manifest.post_filters.include_posted_text] candidates = [ c for c in candidates if not c.posted_text or any(t in (c.posted_text or "").lower() for t in include_lower) ] + if manifest.post_filters.include_title_keywords: + kws = [t.lower() for t in manifest.post_filters.include_title_keywords] + before = len(candidates) + candidates = [ + c for c in candidates + if any(kw in (c.title or "").lower() for kw in kws) + ] + logger.info( + f"[{site_id}] title keyword filter: {len(candidates)}/{before} kept " + f"(keywords: {manifest.post_filters.include_title_keywords})" + ) + # Ensure fingerprints and deduplicate within batch candidates = ensure_fingerprints(candidates) candidates = deduplicate_in_batch(candidates) diff --git a/sites/aroragroup.yaml b/sites/aroragroup.yaml index 2459f2f..bcf0bfb 100644 --- a/sites/aroragroup.yaml +++ b/sites/aroragroup.yaml @@ -27,7 +27,7 @@ pagination: extract: container_selectors: - - ".job-post-row[onclick]" + - ".job-post-row" fields: title: text: diff --git a/sites/global.yaml b/sites/global.yaml index 09f1889..5a7d14c 100644 --- a/sites/global.yaml +++ b/sites/global.yaml @@ -25,5 +25,5 @@ summarization: max_input_items: 200 notification: - provider: kakaotalk + provider: telegram fallback_markdown: true diff --git a/sites/govtjobs.yaml b/sites/govtjobs.yaml index 99ba48f..d5f4514 100644 --- a/sites/govtjobs.yaml +++ b/sites/govtjobs.yaml @@ -19,7 +19,7 @@ search: location_mode: none sort_mode: none date_mode: none - result_list_wait_selector: "h3 a[href*='/jobs/']" + result_list_wait_selector: "" search_override: keywords: diff --git a/sites/linkedin.yaml b/sites/linkedin.yaml index 6be5cc5..24c51fc 100644 --- a/sites/linkedin.yaml +++ b/sites/linkedin.yaml @@ -6,7 +6,7 @@ identity: label: LinkedIn Jobs category: aggregator base_url: https://www.linkedin.com - start_url_template: "https://www.linkedin.com/jobs/search/?keywords={keywords_urlencoded}&f_TPR=r86400&sortBy=DD" + start_url_template: "https://www.linkedin.com/jobs/search/?keywords={keywords_urlencoded}&geoId=103644278&origin=JOB_SEARCH_PAGE_LOCATION_AUTOCOMPLETE" browser: profile_mode: persistent_chrome_profile @@ -19,45 +19,28 @@ search: location_mode: url_param sort_mode: url_param date_mode: url_param - result_list_wait_selector: ".jobs-search__results-list li, .scaffold-layout__list-container li" + result_list_wait_selector: "li[data-occludable-job-id]" pagination: - mode: next_button - next_button_selectors: - - "button[aria-label='View next page']" - - "button[aria-label*='next']" - - "li.artdeco-pagination__indicator--number.selected + li button" - max_pages: 3 + mode: none + max_pages: 1 extract: container_selectors: - - ".jobs-search__results-list li" - - ".scaffold-layout__list-container li" - - "li:has(.base-card)" + - "li[data-occludable-job-id]" fields: title: text: - - "h3.base-search-card__title" - - ".job-card-list__title" - - "h3" - - "a[data-control-name='job_card_title']" + - "a.job-card-list__title--link" company: text: - - "h4.base-search-card__subtitle" - - ".job-card-container__primary-description" - - "a.job-card-container__company-name" + - ".artdeco-entity-lockup__subtitle span" location: text: - - "span.job-search-card__location" - - ".job-card-container__metadata-item" - posted_text: - text: - - "time" - - ".job-search-card__listdate" - - "span[class*='listdate']" + - ".job-card-container__metadata-wrapper li span" url: attr: - selector: "a.base-card__full-link, a[data-control-name='job_card_title'], a[href*='/jobs/view/']" + selector: "a.job-card-list__title--link" name: href post_filters: diff --git a/sites/srpmic.yaml b/sites/srpmic.yaml index 8f90d99..4c51923 100644 --- a/sites/srpmic.yaml +++ b/sites/srpmic.yaml @@ -19,7 +19,7 @@ search: location_mode: none sort_mode: none date_mode: none - result_list_wait_selector: "a[href*='/careers/srpmic/jobs/']" + result_list_wait_selector: "" search_override: keywords: diff --git a/sites/usajobs.yaml b/sites/usajobs.yaml index 596cc55..d364f70 100644 --- a/sites/usajobs.yaml +++ b/sites/usajobs.yaml @@ -64,6 +64,10 @@ search_override: post_filters: include_posted_text: [] + include_title_keywords: + - dentist + - orthodontist + - orthodontic healthcheck: min_items_expected: 0