You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
536 lines
20 KiB
Python
536 lines
20 KiB
Python
"""gimme-job CLI entry point."""
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
import dotenv
|
|
dotenv.load_dotenv()
|
|
|
|
import os
|
|
from loguru import logger as _logger
|
|
_logger.remove()
|
|
_logger.add(sys.stderr, level=os.environ.get("LOG_LEVEL", "INFO"))
|
|
|
|
import typer
|
|
from rich.console import Console
|
|
from rich.panel import Panel
|
|
from rich.table import Table
|
|
|
|
app = typer.Typer(
|
|
name="gimme-job",
|
|
help="Local job posting aggregator — AI-assisted learning, non-AI runtime.",
|
|
add_completion=False,
|
|
)
|
|
console = Console()
|
|
|
|
|
|
# ── init ──────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
@app.command()
|
|
def init(
|
|
force: bool = typer.Option(False, "--force", help="Re-initialize even if already set up"),
|
|
) -> None:
|
|
"""Initialize project directories, database, and config."""
|
|
from gimme_job.db.engine import init_db
|
|
from gimme_job.utils.paths import ensure_workspace_dirs, project_root, sites_dir
|
|
|
|
console.print(Panel("[bold cyan]gimme-job init[/bold cyan]", expand=False))
|
|
|
|
# Create workspace directories
|
|
ensure_workspace_dirs()
|
|
console.print("[green]✓[/green] Workspace directories created")
|
|
|
|
# Create sites/global.yaml if absent
|
|
global_yaml = sites_dir() / "global.yaml"
|
|
if not global_yaml.exists() or force:
|
|
_write_default_global_yaml(global_yaml)
|
|
console.print(f"[green]✓[/green] Created {global_yaml}")
|
|
else:
|
|
console.print(f"[dim] {global_yaml} already exists[/dim]")
|
|
|
|
# Copy .env.example -> .env if absent
|
|
env_example = project_root() / ".env.example"
|
|
env_file = project_root() / ".env"
|
|
if env_example.exists() and not env_file.exists():
|
|
import shutil
|
|
shutil.copy(env_example, env_file)
|
|
console.print(f"[green]✓[/green] Copied .env.example → .env (fill in your tokens)")
|
|
else:
|
|
console.print("[dim] .env already exists[/dim]")
|
|
|
|
# Initialize SQLite DB
|
|
init_db()
|
|
console.print("[green]✓[/green] SQLite database initialized")
|
|
|
|
# Preflight checks
|
|
_check_playwright()
|
|
_check_ollama()
|
|
_check_claude()
|
|
|
|
console.print(Panel("[bold green]Initialization complete.[/bold green]", expand=False))
|
|
|
|
|
|
def _write_default_global_yaml(path: Path) -> None:
|
|
from gimme_job.utils.json_io import write_yaml
|
|
data = {
|
|
"runtime": {
|
|
"timezone": "America/Phoenix",
|
|
"headless": True,
|
|
"profile_name": "JobAgent",
|
|
"slow_mo_ms": 0,
|
|
"default_timeout_ms": 15000,
|
|
"navigation_timeout_ms": 30000,
|
|
"max_pages_per_site": 3,
|
|
"min_delay_ms": 1200,
|
|
"max_delay_ms": 3500,
|
|
},
|
|
"search_defaults": {
|
|
"keywords": ["orthodontist"],
|
|
"location": "",
|
|
"remote": False,
|
|
"date_mode": "today_or_last_24h",
|
|
"sort": "relevance",
|
|
"max_items_per_site": 30,
|
|
},
|
|
"summarization": {
|
|
"ollama_base_url": "http://127.0.0.1:11434",
|
|
"model": "qwen3.5:9b",
|
|
"temperature": 0.1,
|
|
"max_input_items": 200,
|
|
},
|
|
"notification": {
|
|
"provider": "telegram",
|
|
"fallback_markdown": True,
|
|
},
|
|
}
|
|
write_yaml(path, data)
|
|
|
|
|
|
|
|
def _check_playwright() -> None:
|
|
try:
|
|
from playwright.sync_api import sync_playwright
|
|
with sync_playwright() as p:
|
|
_ = p.chromium
|
|
console.print("[green]✓[/green] Playwright available")
|
|
except Exception as e:
|
|
console.print(f"[yellow]![/yellow] Playwright check failed: {e}")
|
|
console.print(" Run: [bold]uv run playwright install chromium[/bold]")
|
|
|
|
|
|
def _check_ollama() -> None:
|
|
try:
|
|
import httpx
|
|
from gimme_job.config import load_global_config
|
|
cfg = load_global_config()
|
|
r = httpx.get(f"{cfg.summarization.ollama_base_url}/api/tags", timeout=3.0)
|
|
if r.status_code == 200:
|
|
console.print("[green]✓[/green] Ollama reachable")
|
|
else:
|
|
console.print(f"[yellow]![/yellow] Ollama returned {r.status_code}")
|
|
except Exception as e:
|
|
console.print(f"[yellow]![/yellow] Ollama not reachable: {e}")
|
|
console.print(" Make sure Ollama is running: [bold]ollama serve[/bold]")
|
|
|
|
|
|
def _check_claude() -> None:
|
|
import subprocess
|
|
try:
|
|
result = subprocess.run(
|
|
["claude", "--version"], capture_output=True, text=True, timeout=10
|
|
)
|
|
if result.returncode == 0:
|
|
console.print(f"[green]✓[/green] Claude Code CLI: {result.stdout.strip()}")
|
|
else:
|
|
console.print("[yellow]![/yellow] Claude Code CLI not found (needed for learn/repair)")
|
|
except FileNotFoundError:
|
|
console.print("[yellow]![/yellow] Claude Code CLI not found (needed for learn/repair)")
|
|
|
|
|
|
# ── login ─────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
@app.command()
|
|
def login() -> None:
|
|
"""Open the JobAgent browser to log in to sites manually.
|
|
|
|
Browse to any sites (LinkedIn, Indeed, Google, etc.), log in, then press Enter.
|
|
All cookies and localStorage are saved in the JobAgent profile and reused on
|
|
every subsequent run.
|
|
"""
|
|
from gimme_job.config import load_global_config
|
|
from gimme_job.runtime.browser import BrowserManager
|
|
|
|
cfg = load_global_config()
|
|
|
|
console.print(Panel("[bold cyan]gimme-job login[/bold cyan]", expand=False))
|
|
console.print(
|
|
"Opening JobAgent Chrome profile.\n"
|
|
"Log in to any sites you need, then press Enter here to close the browser.\n"
|
|
"All cookies and localStorage will be saved and reused on future runs.\n"
|
|
)
|
|
|
|
bm = BrowserManager(
|
|
profile_name=cfg.runtime.profile_name,
|
|
headless=False,
|
|
slow_mo=cfg.runtime.slow_mo_ms,
|
|
)
|
|
try:
|
|
bm.open_context()
|
|
bm.new_page()
|
|
input(" >> Press Enter when done... ")
|
|
finally:
|
|
bm.close()
|
|
|
|
console.print("[green]✓[/green] Browser closed. Session saved to JobAgent profile.")
|
|
|
|
|
|
# ── run ───────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def _execute_run(
|
|
site: Optional[str] = None,
|
|
dry_run: bool = False,
|
|
skip_notify: bool = False,
|
|
limit_sites: Optional[int] = None,
|
|
) -> None:
|
|
"""Core run logic shared by `run` and `auto`."""
|
|
from gimme_job.config import load_global_config
|
|
from gimme_job.db.engine import get_engine, get_session_factory
|
|
from gimme_job.runtime.orchestrator import RunOrchestrator
|
|
|
|
cfg = load_global_config()
|
|
engine = get_engine()
|
|
factory = get_session_factory(engine)
|
|
|
|
orchestrator = RunOrchestrator(global_config=cfg, session_factory=factory)
|
|
summary = orchestrator.run_all(
|
|
site_filter=site,
|
|
dry_run=dry_run,
|
|
skip_notify=skip_notify,
|
|
limit_sites=limit_sites,
|
|
)
|
|
|
|
table = Table(title="Run Summary", show_header=True)
|
|
table.add_column("Site", style="cyan")
|
|
table.add_column("Status")
|
|
table.add_column("Found", justify="right")
|
|
table.add_column("New", justify="right")
|
|
|
|
for r in summary.site_results:
|
|
status_color = {
|
|
"success": "green",
|
|
"partial": "yellow",
|
|
"failed": "red",
|
|
"repair_needed": "red",
|
|
}.get(r.status.value, "white")
|
|
table.add_row(
|
|
r.site_id,
|
|
f"[{status_color}]{r.status.value}[/{status_color}]",
|
|
str(r.items_found),
|
|
str(r.new_items),
|
|
)
|
|
|
|
console.print(table)
|
|
console.print(f"\nTotal: [bold]{summary.total_found}[/bold] found, [bold green]{summary.total_new}[/bold green] new")
|
|
|
|
|
|
@app.command()
|
|
def run(
|
|
site: Optional[str] = typer.Option(None, "--site", help="Run a single site only"),
|
|
dry_run: bool = typer.Option(False, "--dry-run", help="Extract but do not save or notify"),
|
|
skip_notify: bool = typer.Option(False, "--skip-notify", help="Skip Telegram notification"),
|
|
today_only: bool = typer.Option(False, "--today-only", help="Only process today's new items"),
|
|
limit_sites: Optional[int] = typer.Option(None, "--limit-sites", help="Max sites to run"),
|
|
) -> None:
|
|
"""Collect job postings from all enabled sites."""
|
|
_execute_run(site=site, dry_run=dry_run, skip_notify=skip_notify, limit_sites=limit_sites)
|
|
|
|
|
|
# ── auto ──────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
@app.command()
|
|
def auto(
|
|
hour: float = typer.Option(1.0, "--hour", help="Interval in hours between each run (default: 1)"),
|
|
site: Optional[str] = typer.Option(None, "--site", help="Run a single site only"),
|
|
skip_notify: bool = typer.Option(False, "--skip-notify", help="Skip Telegram notification"),
|
|
) -> None:
|
|
"""Run automatically at a fixed interval (default: every 1 hour)."""
|
|
import time
|
|
from datetime import datetime, timedelta
|
|
|
|
interval_secs = int(hour * 3600)
|
|
run_count = 0
|
|
|
|
console.print(Panel(f"[bold cyan]gimme-job auto — every {hour}h[/bold cyan]", expand=False))
|
|
console.print(f"Interval: [bold]{hour}h[/bold]. Press Ctrl+C to stop.\n")
|
|
|
|
try:
|
|
while True:
|
|
run_count += 1
|
|
console.print(
|
|
f"[cyan]── Run #{run_count} {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} ──[/cyan]"
|
|
)
|
|
try:
|
|
_execute_run(site=site, skip_notify=skip_notify)
|
|
except Exception as e:
|
|
console.print(f"[red]Run #{run_count} failed: {e}[/red]")
|
|
|
|
next_run = datetime.now() + timedelta(seconds=interval_secs)
|
|
console.print(
|
|
f"\n[dim]Next run at {next_run.strftime('%H:%M:%S')}. Ctrl+C to stop.[/dim]"
|
|
)
|
|
time.sleep(interval_secs)
|
|
except KeyboardInterrupt:
|
|
console.print("\n[yellow]Auto mode stopped.[/yellow]")
|
|
|
|
|
|
# ── learn ─────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
@app.command()
|
|
def learn(
|
|
site_id: str = typer.Option(..., "--site-id", help="Site identifier (e.g. linkedin)"),
|
|
url: str = typer.Option(..., "--url", help="Job search results URL to learn from"),
|
|
profile_name: str = typer.Option("JobAgent", "--profile-name"),
|
|
headed: bool = typer.Option(True, "--headed/--headless"),
|
|
timeout_seconds: int = typer.Option(180, "--timeout-seconds"),
|
|
) -> None:
|
|
"""Learn a new job site adapter using Claude Code."""
|
|
from gimme_job.config import load_global_config
|
|
from gimme_job.runtime.claude_cli import ClaudeCodeClient
|
|
from gimme_job.runtime.learn import LearnMode
|
|
|
|
cfg = load_global_config()
|
|
client = ClaudeCodeClient()
|
|
mode = LearnMode(global_config=cfg, claude_client=client)
|
|
|
|
console.print(Panel(f"[bold cyan]Learning site: {site_id}[/bold cyan]", expand=False))
|
|
success = mode.learn_site(
|
|
site_id=site_id,
|
|
url=url,
|
|
profile_name=profile_name,
|
|
headed=headed,
|
|
timeout=timeout_seconds,
|
|
)
|
|
|
|
if success:
|
|
console.print(f"[green]✓[/green] Successfully learned [bold]{site_id}[/bold]")
|
|
else:
|
|
console.print(f"[red]✗[/red] Failed to learn [bold]{site_id}[/bold]")
|
|
raise typer.Exit(1)
|
|
|
|
|
|
# ── repair ────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
@app.command()
|
|
def repair(
|
|
site: Optional[str] = typer.Argument(None, help="Site ID to repair"),
|
|
all_: bool = typer.Option(False, "--all", help="Repair all repair_needed sites"),
|
|
) -> None:
|
|
"""Repair a broken site adapter using Claude Code."""
|
|
from gimme_job.config import load_global_config, list_all_sites
|
|
from gimme_job.runtime.claude_cli import ClaudeCodeClient
|
|
from gimme_job.runtime.repair import RepairMode
|
|
|
|
if not site and not all_:
|
|
console.print("[red]Provide a site ID or --all[/red]")
|
|
raise typer.Exit(1)
|
|
|
|
cfg = load_global_config()
|
|
client = ClaudeCodeClient()
|
|
mode = RepairMode(global_config=cfg, claude_client=client)
|
|
|
|
sites_to_repair = list_all_sites() if all_ else [site]
|
|
|
|
for s in sites_to_repair:
|
|
console.print(f"[cyan]Repairing {s}...[/cyan]")
|
|
success = mode.repair_site(s)
|
|
if success:
|
|
console.print(f"[green]✓[/green] {s} repaired")
|
|
else:
|
|
console.print(f"[red]✗[/red] {s} repair failed")
|
|
|
|
|
|
# ── test ──────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
@app.command()
|
|
def test(
|
|
site: str = typer.Argument(..., help="Site ID to test"),
|
|
) -> None:
|
|
"""Run smoke test for a site adapter."""
|
|
import subprocess
|
|
|
|
console.print(Panel(f"[bold cyan]Testing: {site}[/bold cyan]", expand=False))
|
|
result = subprocess.run(
|
|
["uv", "run", "pytest", f"tests/adapters/test_{site}.py", "-v"],
|
|
cwd=str(Path(__file__).parent.parent),
|
|
)
|
|
raise typer.Exit(result.returncode)
|
|
|
|
|
|
# ── notify ────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
@app.command()
|
|
def notify(
|
|
today: bool = typer.Option(True, "--today/--no-today", help="Re-send today's job listing"),
|
|
) -> None:
|
|
"""Send today's new job listings via Telegram (one message per site group)."""
|
|
from datetime import date
|
|
|
|
from gimme_job.config import load_global_config
|
|
from gimme_job.db.engine import get_engine, get_session_factory
|
|
from gimme_job.db.repo import JobPostingRepo
|
|
from gimme_job.runtime.notifier import NotificationDispatcher, build_listing_messages
|
|
from gimme_job.runtime.telegram import TelegramClient
|
|
|
|
cfg = load_global_config()
|
|
engine = get_engine()
|
|
factory = get_session_factory(engine)
|
|
run_date = date.today()
|
|
|
|
with factory() as session:
|
|
postings = JobPostingRepo().get_today_new(session, run_date)
|
|
|
|
if not postings:
|
|
console.print("[yellow]No new postings found for today.[/yellow]")
|
|
raise typer.Exit(0)
|
|
|
|
chunks = build_listing_messages(postings)
|
|
console.print(f"Sending {len(chunks)} message(s) for {len(postings)} postings...")
|
|
|
|
client = TelegramClient()
|
|
sent = 0
|
|
if client.is_configured():
|
|
for chunk in chunks:
|
|
if client.send_message(chunk):
|
|
sent += 1
|
|
if sent == len(chunks):
|
|
console.print(f"[green]✓[/green] All {sent} message(s) sent")
|
|
else:
|
|
console.print(f"[yellow]![/yellow] Sent {sent}/{len(chunks)} messages")
|
|
else:
|
|
console.print("[yellow]Telegram not configured[/yellow]")
|
|
|
|
# Always write markdown fallback
|
|
dispatcher = NotificationDispatcher(global_config=cfg, session_factory=factory)
|
|
full_text = "\n\n".join(chunks)
|
|
path = dispatcher.send_markdown_fallback(full_text, run_date)
|
|
console.print(f"Markdown saved: {path}")
|
|
|
|
|
|
# ── list ──────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
@app.command(name="list")
|
|
def list_postings(
|
|
site: Optional[str] = typer.Option(None, "--site", help="Filter by site ID"),
|
|
today: bool = typer.Option(False, "--today", help="Only show today's new postings"),
|
|
limit: int = typer.Option(50, "--limit", help="Max rows to show"),
|
|
) -> None:
|
|
"""List collected job postings from the database."""
|
|
from datetime import date
|
|
|
|
from gimme_job.db.engine import get_engine, get_session_factory
|
|
from gimme_job.db.repo import JobPostingRepo
|
|
from gimme_job.models.db import JobPosting
|
|
from sqlalchemy import select
|
|
|
|
engine = get_engine()
|
|
factory = get_session_factory(engine)
|
|
|
|
table = Table(title="Job Postings", show_header=True, show_lines=False)
|
|
table.add_column("#", style="dim", justify="right", width=4)
|
|
table.add_column("Site", style="cyan", width=10)
|
|
table.add_column("Title", width=40)
|
|
table.add_column("Company", width=25)
|
|
table.add_column("Location", width=20)
|
|
table.add_column("Posted", width=12)
|
|
table.add_column("New", width=4)
|
|
|
|
with factory() as session:
|
|
q = select(JobPosting).order_by(JobPosting.first_seen_at.desc())
|
|
if site:
|
|
q = q.where(JobPosting.site_id == site)
|
|
if today:
|
|
q = q.where(JobPosting.run_date == date.today(), JobPosting.is_new == True)
|
|
q = q.limit(limit)
|
|
rows = list(session.execute(q).scalars().all())
|
|
|
|
for i, p in enumerate(rows, 1):
|
|
table.add_row(
|
|
str(i),
|
|
p.site_id,
|
|
p.title[:38] if p.title else "",
|
|
(p.company or "")[:23],
|
|
(p.location or "")[:18],
|
|
p.posted_text or "",
|
|
"[green]Y[/green]" if p.is_new else "",
|
|
)
|
|
|
|
console.print(table)
|
|
console.print(f"[dim]{len(rows)} rows (limit={limit})[/dim]")
|
|
|
|
|
|
# ── status ────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
@app.command()
|
|
def status() -> None:
|
|
"""Show health status of all configured sites."""
|
|
from gimme_job.config import list_all_sites, load_site_manifest
|
|
from gimme_job.db.engine import get_engine, get_session_factory
|
|
from gimme_job.db.repo import SiteConfigRepo, SiteRunRepo
|
|
|
|
engine = get_engine()
|
|
factory = get_session_factory(engine)
|
|
|
|
from sqlalchemy import func, select
|
|
from gimme_job.models.db import JobPosting
|
|
|
|
table = Table(title="Site Status", show_header=True)
|
|
table.add_column("Site", style="cyan")
|
|
table.add_column("Enabled")
|
|
table.add_column("Repair?")
|
|
table.add_column("Stored", justify="right")
|
|
table.add_column("Last Run")
|
|
table.add_column("Last Status")
|
|
table.add_column("Failures", justify="right")
|
|
|
|
run_repo = SiteRunRepo()
|
|
config_repo = SiteConfigRepo()
|
|
|
|
with factory() as session:
|
|
for site_id in list_all_sites():
|
|
try:
|
|
manifest = load_site_manifest(site_id)
|
|
except Exception:
|
|
continue
|
|
|
|
config = session.get(__import__("gimme_job.models.db", fromlist=["SiteConfig"]).SiteConfig, site_id)
|
|
recent = run_repo.get_recent_runs(session, site_id, limit=1)
|
|
stored = session.execute(
|
|
select(func.count()).where(JobPosting.site_id == site_id)
|
|
).scalar() or 0
|
|
|
|
enabled = "[green]yes[/green]" if manifest.enabled else "[dim]no[/dim]"
|
|
repair = "[red]YES[/red]" if manifest.repair_needed else "[green]no[/green]"
|
|
last_run = recent[0].started_at.strftime("%m-%d %H:%M") if recent else "[dim]never[/dim]"
|
|
last_status = recent[0].status if recent else "[dim]-[/dim]"
|
|
failures = str(config.consecutive_failures) if config else "0"
|
|
|
|
table.add_row(site_id, enabled, repair, str(stored), last_run, last_status, failures)
|
|
|
|
console.print(table)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
app()
|