from __future__ import annotations from datetime import datetime from pathlib import Path from typing import Any, Optional from pydantic import BaseModel, Field class SiteIdentity(BaseModel): label: str category: str = "aggregator" base_url: str start_url_template: str class BrowserConfig(BaseModel): profile_mode: str = "persistent_chrome_profile" profile_name: str = "JobAgent" headed_on_learn: bool = True headless_on_run: bool = True class SearchConfig(BaseModel): keyword_mode: str = "url_param" # url_param | input_field | none location_mode: str = "url_param" # url_param | input_field | none sort_mode: str = "none" date_mode: str = "none" # none | url_param | click_filter date_filter_text: Optional[str] = None result_list_wait_selector: Optional[str] = None class PaginationConfig(BaseModel): mode: str = "none" # none | next_button | url_increment | scroll next_button_selectors: list[str] = Field(default_factory=list) max_pages: int = 3 class AttrSelector(BaseModel): selector: str name: str # attribute name, e.g. "href" class FieldSelector(BaseModel): text: Optional[list[str]] = None # CSS selectors to try for inner_text() attr: Optional[AttrSelector] = None # CSS selector + attribute name class ExtractConfig(BaseModel): container_selectors: list[str] = Field(default_factory=list) fields: dict[str, FieldSelector] = Field(default_factory=dict) class PostFilterConfig(BaseModel): include_posted_text: list[str] = Field(default_factory=list) class HealthcheckConfig(BaseModel): min_items_expected: int = 0 required_fields: list[str] = Field(default_factory=lambda: ["title", "url"]) fail_if_zero_when_known_active: bool = False class SearchOverride(BaseModel): """Site-level search parameter overrides.""" keywords: Optional[list[str]] = None location: Optional[str] = None remote: Optional[bool] = None date_mode: Optional[str] = None sort: Optional[str] = None max_items: Optional[int] = None multi_keyword_mode: Optional[str] = None # "separate" → one scrape per keyword class LearnMeta(BaseModel): last_learned_at: Optional[datetime] = None source_url: Optional[str] = None class SiteManifest(BaseModel): site_id: str enabled: bool = True repair_needed: bool = False identity: SiteIdentity browser: BrowserConfig = Field(default_factory=BrowserConfig) search: SearchConfig = Field(default_factory=SearchConfig) search_override: SearchOverride = Field(default_factory=SearchOverride) pagination: PaginationConfig = Field(default_factory=PaginationConfig) extract: ExtractConfig = Field(default_factory=ExtractConfig) post_filters: PostFilterConfig = Field(default_factory=PostFilterConfig) healthcheck: HealthcheckConfig = Field(default_factory=HealthcheckConfig) learn: LearnMeta = Field(default_factory=LearnMeta) @classmethod def from_yaml(cls, path: Path) -> "SiteManifest": from gimme_job.utils.json_io import read_yaml data = read_yaml(path) return cls.model_validate(data) def to_yaml(self, path: Path) -> None: from gimme_job.utils.json_io import write_yaml write_yaml(path, self.model_dump(mode="json")) def build_start_url(self, query: Any) -> str: """Render the start URL template with query parameters.""" template = self.identity.start_url_template try: return template.format( keywords_urlencoded=query.keywords_urlencoded, keywords=query.keywords_str, location_urlencoded=query.location_urlencoded, location=query.location, ) except KeyError: return template