You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
108 lines
3.8 KiB
Python
108 lines
3.8 KiB
Python
"""
|
|
Wikimedia REST API adapter - collect daily page view counts for watched pages.
|
|
"""
|
|
|
|
import logging
|
|
from datetime import datetime, timedelta, timezone
|
|
from typing import List, Optional
|
|
|
|
import aiohttp
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
from sqlalchemy import select, and_
|
|
|
|
from app.models.overlay_raw_event import OverlayWikiPageview
|
|
from app.models.overlay_registry import WikiPageMap
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
WIKIMEDIA_BASE = "https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article"
|
|
USER_AGENT = "StockOracle/1.0 (github.com/stockoracle; contact@stockoracle.com)"
|
|
|
|
|
|
class WikimediaAdapter:
|
|
"""Collect Wikimedia page view data for pages mapped to tickers."""
|
|
|
|
async def _get_watched_pages(self, db: AsyncSession) -> List[WikiPageMap]:
|
|
result = await db.execute(
|
|
select(WikiPageMap).where(WikiPageMap.active == True)
|
|
)
|
|
return result.scalars().all()
|
|
|
|
async def fetch_pageviews(
|
|
self, page_title: str, date: datetime, project: str = "en.wikipedia"
|
|
) -> Optional[int]:
|
|
"""Fetch daily pageviews for a specific page and date."""
|
|
date_str = date.strftime("%Y%m%d")
|
|
encoded_title = page_title.replace(" ", "_")
|
|
url = (
|
|
f"{WIKIMEDIA_BASE}/{project}/all-access/all-agents"
|
|
f"/{encoded_title}/daily/{date_str}/{date_str}"
|
|
)
|
|
try:
|
|
headers = {"User-Agent": USER_AGENT}
|
|
async with aiohttp.ClientSession(headers=headers) as session:
|
|
async with session.get(url, timeout=aiohttp.ClientTimeout(total=30)) as resp:
|
|
if resp.status == 404:
|
|
return None
|
|
resp.raise_for_status()
|
|
data = await resp.json()
|
|
items = data.get("items", [])
|
|
if items:
|
|
return items[0].get("views", 0)
|
|
return 0
|
|
except Exception as e:
|
|
logger.debug(f"Wikimedia fetch error for '{page_title}' on {date_str}: {e}")
|
|
return None
|
|
|
|
async def collect(self, db: AsyncSession, days_back: int = 3) -> int:
|
|
"""
|
|
Collect pageviews for all active wiki page mappings.
|
|
|
|
Returns number of new records inserted.
|
|
"""
|
|
pages = await self._get_watched_pages(db)
|
|
if not pages:
|
|
logger.info("Wikimedia: no active wiki page mappings found")
|
|
return 0
|
|
|
|
inserted = 0
|
|
now = datetime.now(timezone.utc)
|
|
|
|
for page in pages:
|
|
for days_ago in range(1, days_back + 1):
|
|
target_date = now - timedelta(days=days_ago)
|
|
target_date = target_date.replace(hour=0, minute=0, second=0, microsecond=0)
|
|
|
|
# Check duplicate
|
|
existing = await db.execute(
|
|
select(OverlayWikiPageview.id).where(
|
|
and_(
|
|
OverlayWikiPageview.page_title == page.wiki_page_title,
|
|
OverlayWikiPageview.date == target_date,
|
|
OverlayWikiPageview.project == "en.wikipedia",
|
|
)
|
|
)
|
|
)
|
|
if existing.first():
|
|
continue
|
|
|
|
views = await self.fetch_pageviews(page.wiki_page_title, target_date)
|
|
if views is None:
|
|
continue
|
|
|
|
record = OverlayWikiPageview(
|
|
page_title=page.wiki_page_title,
|
|
date=target_date,
|
|
project="en.wikipedia",
|
|
views=views,
|
|
mapped_symbol=page.symbol,
|
|
)
|
|
db.add(record)
|
|
inserted += 1
|
|
|
|
if inserted:
|
|
await db.commit()
|
|
logger.info(f"Wikimedia: inserted {inserted} pageview records")
|
|
|
|
return inserted
|