""" Wikimedia REST API adapter - collect daily page view counts for watched pages. """ import logging from datetime import datetime, timedelta, timezone from typing import List, Optional import aiohttp from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy import select, and_ from app.models.overlay_raw_event import OverlayWikiPageview from app.models.overlay_registry import WikiPageMap logger = logging.getLogger(__name__) WIKIMEDIA_BASE = "https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article" USER_AGENT = "StockOracle/1.0 (github.com/stockoracle; contact@stockoracle.com)" class WikimediaAdapter: """Collect Wikimedia page view data for pages mapped to tickers.""" async def _get_watched_pages(self, db: AsyncSession) -> List[WikiPageMap]: result = await db.execute( select(WikiPageMap).where(WikiPageMap.active == True) ) return result.scalars().all() async def fetch_pageviews( self, page_title: str, date: datetime, project: str = "en.wikipedia" ) -> Optional[int]: """Fetch daily pageviews for a specific page and date.""" date_str = date.strftime("%Y%m%d") encoded_title = page_title.replace(" ", "_") url = ( f"{WIKIMEDIA_BASE}/{project}/all-access/all-agents" f"/{encoded_title}/daily/{date_str}/{date_str}" ) try: headers = {"User-Agent": USER_AGENT} async with aiohttp.ClientSession(headers=headers) as session: async with session.get(url, timeout=aiohttp.ClientTimeout(total=30)) as resp: if resp.status == 404: return None resp.raise_for_status() data = await resp.json() items = data.get("items", []) if items: return items[0].get("views", 0) return 0 except Exception as e: logger.debug(f"Wikimedia fetch error for '{page_title}' on {date_str}: {e}") return None async def collect(self, db: AsyncSession, days_back: int = 3) -> int: """ Collect pageviews for all active wiki page mappings. Returns number of new records inserted. """ pages = await self._get_watched_pages(db) if not pages: logger.info("Wikimedia: no active wiki page mappings found") return 0 inserted = 0 now = datetime.now(timezone.utc) for page in pages: for days_ago in range(1, days_back + 1): target_date = now - timedelta(days=days_ago) target_date = target_date.replace(hour=0, minute=0, second=0, microsecond=0) # Check duplicate existing = await db.execute( select(OverlayWikiPageview.id).where( and_( OverlayWikiPageview.page_title == page.wiki_page_title, OverlayWikiPageview.date == target_date, OverlayWikiPageview.project == "en.wikipedia", ) ) ) if existing.first(): continue views = await self.fetch_pageviews(page.wiki_page_title, target_date) if views is None: continue record = OverlayWikiPageview( page_title=page.wiki_page_title, date=target_date, project="en.wikipedia", views=views, mapped_symbol=page.symbol, ) db.add(record) inserted += 1 if inserted: await db.commit() logger.info(f"Wikimedia: inserted {inserted} pageview records") return inserted