You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
88 lines
2.8 KiB
Python
88 lines
2.8 KiB
Python
"""
|
|
Overlay pipeline orchestrator — runs the Yahoo RSS headline collector.
|
|
"""
|
|
|
|
import logging
|
|
from datetime import datetime, timezone
|
|
from typing import Dict, List, Optional
|
|
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
from sqlalchemy import select
|
|
|
|
from app.models.overlay_feature import OverlayJobLog
|
|
from app.services.overlay.yahoo_rss_adapter import YahooRSSAdapter
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class OverlayPipeline:
|
|
"""Orchestrate Yahoo RSS headline collection."""
|
|
|
|
def __init__(self):
|
|
self.rss = YahooRSSAdapter()
|
|
|
|
async def _log_job(
|
|
self,
|
|
db: AsyncSession,
|
|
job_type: str,
|
|
status: str,
|
|
started_at: datetime,
|
|
records: int = 0,
|
|
error: Optional[str] = None,
|
|
) -> None:
|
|
if status != "running":
|
|
existing_result = await db.execute(
|
|
select(OverlayJobLog)
|
|
.where(
|
|
OverlayJobLog.job_type == job_type,
|
|
OverlayJobLog.started_at == started_at,
|
|
OverlayJobLog.status == "running",
|
|
)
|
|
.limit(1)
|
|
)
|
|
existing = existing_result.scalars().first()
|
|
if existing:
|
|
existing.status = status
|
|
existing.completed_at = datetime.now(timezone.utc)
|
|
existing.records_processed = records
|
|
existing.error_message = error
|
|
try:
|
|
await db.commit()
|
|
except Exception as e:
|
|
logger.warning(f"Failed to update job log: {e}")
|
|
await db.rollback()
|
|
return
|
|
|
|
log = OverlayJobLog(
|
|
job_type=job_type,
|
|
status=status,
|
|
started_at=started_at,
|
|
completed_at=datetime.now(timezone.utc) if status != "running" else None,
|
|
records_processed=records,
|
|
error_message=error,
|
|
)
|
|
db.add(log)
|
|
try:
|
|
await db.commit()
|
|
except Exception as e:
|
|
logger.warning(f"Failed to persist job log: {e}")
|
|
await db.rollback()
|
|
|
|
async def collect_all(self, db: AsyncSession) -> Dict:
|
|
"""Run the Yahoo RSS headline collector."""
|
|
started = datetime.now(timezone.utc)
|
|
await self._log_job(db, "collect_all", "running", started)
|
|
|
|
counts: Dict[str, int] = {"yahoo_rss": 0}
|
|
error: Optional[str] = None
|
|
|
|
try:
|
|
counts["yahoo_rss"] = await self.rss.collect(db)
|
|
except Exception as e:
|
|
logger.error(f"yahoo_rss collect error: {e}")
|
|
error = str(e)
|
|
|
|
final_status = "partial" if error else "completed"
|
|
await self._log_job(db, "collect_all", final_status, started, sum(counts.values()), error)
|
|
return counts
|