You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
97 lines
3.1 KiB
Python
97 lines
3.1 KiB
Python
"""
|
|
Google Trends adapter (experimental, feature-flagged).
|
|
Disabled by default; enable via GOOGLE_TRENDS_ENABLED=true env var.
|
|
"""
|
|
|
|
import logging
|
|
from datetime import datetime, timezone
|
|
from typing import List
|
|
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
from sqlalchemy import select, and_
|
|
|
|
from app.core.config import settings
|
|
from app.models.overlay_registry import ThemeTopicMap
|
|
from app.models.overlay_raw_event import OverlayTrendObservation
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class GoogleTrendsAdapter:
|
|
"""Collect Google Trends data (experimental, disabled by default)."""
|
|
|
|
@property
|
|
def enabled(self) -> bool:
|
|
return getattr(settings, "GOOGLE_TRENDS_ENABLED", False)
|
|
|
|
async def collect(self, db: AsyncSession) -> int:
|
|
"""
|
|
Collect Google Trends interest data for active topics.
|
|
|
|
Returns 0 if feature flag is disabled or pytrends is not installed.
|
|
Returns number of new records inserted.
|
|
"""
|
|
if not self.enabled:
|
|
logger.debug("Google Trends adapter: feature flag disabled, skipping")
|
|
return 0
|
|
|
|
try:
|
|
from pytrends.request import TrendReq
|
|
except ImportError:
|
|
logger.warning("pytrends not installed; Google Trends adapter inactive")
|
|
return 0
|
|
|
|
result = await db.execute(
|
|
select(ThemeTopicMap).where(ThemeTopicMap.active == True)
|
|
)
|
|
topics = result.scalars().all()
|
|
if not topics:
|
|
return 0
|
|
|
|
inserted = 0
|
|
|
|
try:
|
|
pytrends = TrendReq(hl="en-US", tz=360)
|
|
# pytrends limits to 5 keywords at a time
|
|
kw_list = [t.topic_label for t in topics[:5]]
|
|
pytrends.build_payload(kw_list, timeframe="now 7-d", geo="US")
|
|
interest_df = pytrends.interest_over_time()
|
|
except Exception as e:
|
|
logger.error(f"Google Trends API error: {e}")
|
|
return 0
|
|
|
|
for topic in topics:
|
|
if topic.topic_label not in interest_df.columns:
|
|
continue
|
|
series = interest_df[topic.topic_label]
|
|
for ts, val in series.items():
|
|
observed_at = ts.to_pydatetime().replace(tzinfo=timezone.utc)
|
|
|
|
# Check duplicate
|
|
existing = await db.execute(
|
|
select(OverlayTrendObservation.id).where(
|
|
and_(
|
|
OverlayTrendObservation.topic_id == topic.topic_id,
|
|
OverlayTrendObservation.observed_at == observed_at,
|
|
OverlayTrendObservation.geography == "US",
|
|
)
|
|
)
|
|
)
|
|
if existing.first():
|
|
continue
|
|
|
|
record = OverlayTrendObservation(
|
|
topic_id=topic.topic_id,
|
|
observed_at=observed_at,
|
|
geography="US",
|
|
interest_value=int(val),
|
|
)
|
|
db.add(record)
|
|
inserted += 1
|
|
|
|
if inserted:
|
|
await db.commit()
|
|
logger.info(f"Google Trends: inserted {inserted} observations")
|
|
|
|
return inserted
|