You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
58 lines
1.8 KiB
Python
58 lines
1.8 KiB
Python
"""Site health tracking and repair_needed logic."""
|
|
from __future__ import annotations
|
|
|
|
from typing import Optional
|
|
|
|
from loguru import logger
|
|
from sqlalchemy.orm import Session
|
|
|
|
from gimme_job.constants import CONSECUTIVE_FAILURES_THRESHOLD, FailureClassification, RunStatus
|
|
from gimme_job.models.manifest import SiteManifest
|
|
from gimme_job.models.runtime import SiteRunResult
|
|
|
|
|
|
ANTI_BOT_PATTERNS = [
|
|
"access denied",
|
|
"captcha",
|
|
"robot",
|
|
"are you human",
|
|
"unusual activity",
|
|
"blocked",
|
|
"403",
|
|
"cloudflare",
|
|
]
|
|
|
|
|
|
def classify_page_content(html: str) -> Optional[FailureClassification]:
|
|
"""Detect anti-bot or login-required signals in page HTML."""
|
|
html_lower = html.lower()
|
|
if any(p in html_lower for p in ANTI_BOT_PATTERNS):
|
|
return FailureClassification.ANTI_BOT_SUSPECTED
|
|
login_signals = ["sign in", "log in", "login required", "please log in"]
|
|
if any(p in html_lower for p in login_signals):
|
|
return FailureClassification.LOGIN_REQUIRED
|
|
return None
|
|
|
|
|
|
def update_site_status(
|
|
site_id: str,
|
|
result: SiteRunResult,
|
|
session: Session,
|
|
) -> None:
|
|
"""Update SiteConfig table based on run result."""
|
|
from gimme_job.db.repo import SiteConfigRepo
|
|
|
|
repo = SiteConfigRepo()
|
|
|
|
if result.status == RunStatus.SUCCESS:
|
|
repo.record_success(session, site_id)
|
|
logger.info(f"[{site_id}] Run success — failures reset")
|
|
elif result.status == RunStatus.FAILED:
|
|
count = repo.increment_failure(session, site_id)
|
|
logger.warning(f"[{site_id}] Run failed — consecutive failures: {count}")
|
|
if count >= CONSECUTIVE_FAILURES_THRESHOLD:
|
|
logger.error(f"[{site_id}] repair_needed set after {count} consecutive failures")
|
|
elif result.status == RunStatus.PARTIAL:
|
|
# Partial success — don't increment failures, but don't reset either
|
|
pass
|