You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
117 lines
3.6 KiB
Python
117 lines
3.6 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import Any, Optional
|
|
|
|
from pydantic import BaseModel, Field
|
|
|
|
|
|
class SiteIdentity(BaseModel):
|
|
label: str
|
|
category: str = "aggregator"
|
|
base_url: str
|
|
start_url_template: str
|
|
|
|
|
|
class BrowserConfig(BaseModel):
|
|
profile_mode: str = "persistent_chrome_profile"
|
|
profile_name: str = "JobAgent"
|
|
headed_on_learn: bool = True
|
|
headless_on_run: bool = True
|
|
|
|
|
|
class SearchConfig(BaseModel):
|
|
keyword_mode: str = "url_param" # url_param | input_field | none
|
|
location_mode: str = "url_param" # url_param | input_field | none
|
|
sort_mode: str = "none"
|
|
date_mode: str = "none" # none | url_param | click_filter
|
|
date_filter_text: Optional[str] = None
|
|
result_list_wait_selector: Optional[str] = None
|
|
|
|
|
|
class PaginationConfig(BaseModel):
|
|
mode: str = "none" # none | next_button | url_increment | scroll
|
|
next_button_selectors: list[str] = Field(default_factory=list)
|
|
max_pages: int = 3
|
|
|
|
|
|
class AttrSelector(BaseModel):
|
|
selector: str
|
|
name: str # attribute name, e.g. "href"
|
|
|
|
|
|
class FieldSelector(BaseModel):
|
|
text: Optional[list[str]] = None # CSS selectors to try for inner_text()
|
|
attr: Optional[AttrSelector] = None # CSS selector + attribute name
|
|
|
|
|
|
class ExtractConfig(BaseModel):
|
|
container_selectors: list[str] = Field(default_factory=list)
|
|
fields: dict[str, FieldSelector] = Field(default_factory=dict)
|
|
|
|
|
|
class PostFilterConfig(BaseModel):
|
|
include_posted_text: list[str] = Field(default_factory=list)
|
|
|
|
|
|
class HealthcheckConfig(BaseModel):
|
|
min_items_expected: int = 0
|
|
required_fields: list[str] = Field(default_factory=lambda: ["title", "url"])
|
|
fail_if_zero_when_known_active: bool = False
|
|
|
|
|
|
class SearchOverride(BaseModel):
|
|
"""Site-level search parameter overrides."""
|
|
|
|
keywords: Optional[list[str]] = None
|
|
location: Optional[str] = None
|
|
remote: Optional[bool] = None
|
|
date_mode: Optional[str] = None
|
|
sort: Optional[str] = None
|
|
max_items: Optional[int] = None
|
|
|
|
|
|
class LearnMeta(BaseModel):
|
|
last_learned_at: Optional[datetime] = None
|
|
source_url: Optional[str] = None
|
|
|
|
|
|
class SiteManifest(BaseModel):
|
|
site_id: str
|
|
enabled: bool = True
|
|
repair_needed: bool = False
|
|
|
|
identity: SiteIdentity
|
|
browser: BrowserConfig = Field(default_factory=BrowserConfig)
|
|
search: SearchConfig = Field(default_factory=SearchConfig)
|
|
search_override: SearchOverride = Field(default_factory=SearchOverride)
|
|
pagination: PaginationConfig = Field(default_factory=PaginationConfig)
|
|
extract: ExtractConfig = Field(default_factory=ExtractConfig)
|
|
post_filters: PostFilterConfig = Field(default_factory=PostFilterConfig)
|
|
healthcheck: HealthcheckConfig = Field(default_factory=HealthcheckConfig)
|
|
learn: LearnMeta = Field(default_factory=LearnMeta)
|
|
|
|
@classmethod
|
|
def from_yaml(cls, path: Path) -> "SiteManifest":
|
|
from gimme_job.utils.json_io import read_yaml
|
|
data = read_yaml(path)
|
|
return cls.model_validate(data)
|
|
|
|
def to_yaml(self, path: Path) -> None:
|
|
from gimme_job.utils.json_io import write_yaml
|
|
write_yaml(path, self.model_dump(mode="json"))
|
|
|
|
def build_start_url(self, query: Any) -> str:
|
|
"""Render the start URL template with query parameters."""
|
|
template = self.identity.start_url_template
|
|
try:
|
|
return template.format(
|
|
keywords_urlencoded=query.keywords_urlencoded,
|
|
keywords=query.keywords_str,
|
|
location_urlencoded=query.location_urlencoded,
|
|
location=query.location,
|
|
)
|
|
except KeyError:
|
|
return template
|