proactive: circuit-breaker for blocked engines, track blocks separately

Google Jobs anti-bot wall fired on every query from Korea, flooding the
summary with errors. Now an engine that raises EngineBlockedError is
skipped for the rest of the run (one attempt total), and blocks are
counted separately instead of polluting the error list.
main
I Luk Kim 4 weeks ago
parent 05510274a3
commit e98f91036c

@ -476,12 +476,18 @@ def proactive_run(
("CLOSED", result.closed_count),
("STALE", result.stale_count),
("Filtered (non-ortho)", result.filtered_count),
("Engine blocks", result.engine_blocks),
]
for metric, value in rows:
table.add_row(metric, str(value))
console.print(table)
if result.engine_blocks:
console.print(
f"[yellow]⚠ {result.engine_blocks} engine block(s) — "
f"anti-bot으로 차단된 검색엔진은 이번 런에서 건너뜀[/yellow]"
)
if result.errors:
console.print(f"[yellow]{len(result.errors)} error(s) logged — see run logs[/yellow]")
console.print(f"[red]{len(result.errors)} error(s) logged — see run logs[/red]")
@proactive_app.command("auto")

@ -42,6 +42,7 @@ class ProactiveRunResult:
closed_count: int = 0
stale_count: int = 0
filtered_count: int = 0
engine_blocks: int = 0
errors: list[str] = field(default_factory=list)
def finish(self) -> None:
@ -116,7 +117,7 @@ class ProactiveEngine:
f"NEW={result.new_count} REOPENED={result.reopened_count} "
f"STILL_OPEN={result.still_open_count} VERIFY={result.verify_count} "
f"CLOSED={result.closed_count} STALE={result.stale_count} "
f"filtered={result.filtered_count}"
f"filtered={result.filtered_count} blocks={result.engine_blocks}"
)
return result
@ -131,9 +132,12 @@ class ProactiveEngine:
) -> list[SearchResult]:
discovered: list[SearchResult] = []
seen_urls: set[str] = set()
blocked_engines: set[str] = set()
for query in queries:
for engine_name in plan.enabled_engines():
if engine_name in blocked_engines:
continue # circuit breaker: skip engines blocked this run
engine = build_engine(engine_name)
cfg = plan.engine_config(engine_name)
try:
@ -147,8 +151,13 @@ class ProactiveEngine:
seen_urls.add(canon)
discovered.append(r)
except EngineBlockedError as e:
logger.warning(f"[proactive] {engine_name} blocked: {e}")
result.errors.append(str(e))
# Engine anti-bot wall — not a real error; skip engine for rest of run
blocked_engines.add(engine_name)
result.engine_blocks += 1
logger.warning(
f"[proactive] {engine_name} blocked ({e}) — "
f"skipping it for the rest of this run"
)
except Exception as e:
logger.warning(f"[proactive] {engine_name} query failed: {e}")
result.errors.append(str(e)[:200])

Loading…
Cancel
Save