stock-oracle/app/services/overlay/yahoo_rss_adapter.py

"""
General-purpose financial news RSS adapter.

Pulls a small set of live general feeds (Yahoo Finance, CNBC, MarketWatch,
Seeking Alpha) and runs every headline through the EntityResolver to attach
matched ticker symbols. Replaces the previous per-symbol pull strategy.
"""

import asyncio
import logging
from datetime import datetime, timezone
from typing import Dict, List, Optional
from urllib.parse import urlparse

import aiohttp
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select

from app.core.http_client import get_http_session
from app.models.overlay_raw_event import OverlayHeadlineEvent
from app.services.overlay.entity_resolver import EntityResolver

logger = logging.getLogger(__name__)


# General financial-news feeds. Each one returns a stream of latest headlines;
# we let EntityResolver decide which tickers each headline mentions.
_GENERAL_FEEDS: List[str] = [
    "https://finance.yahoo.com/news/rssindex",
    "https://www.cnbc.com/id/100003114/device/rss/rss.html",   # CNBC Top News
    "https://www.cnbc.com/id/15839135/device/rss/rss.html",    # CNBC Markets
    "https://feeds.content.dowjones.io/public/rss/mw_topstories",
    "https://seekingalpha.com/market_currents.xml",
]


_PUBLISHER_DOMAINS: Dict[str, str] = {
    "fool.com": "Motley Fool",
    "finance.yahoo.com": "Yahoo Finance",
    "news.yahoo.com": "Yahoo News",
    "yahoo.com": "Yahoo Finance",
    "bloomberg.com": "Bloomberg",
    "reuters.com": "Reuters",
    "wsj.com": "Wall Street Journal",
    "cnbc.com": "CNBC",
    "marketwatch.com": "MarketWatch",
    "barrons.com": "Barron's",
    "ft.com": "Financial Times",
    "investors.com": "Investor's Business Daily",
    "forbes.com": "Forbes",
    "businessinsider.com": "Business Insider",
    "seekingalpha.com": "Seeking Alpha",
    "zacks.com": "Zacks",
    "thestreet.com": "TheStreet",
    "benzinga.com": "Benzinga",
    "investorplace.com": "InvestorPlace",
    "morningstar.com": "Morningstar",
    "investopedia.com": "Investopedia",
    "businesswire.com": "Business Wire",
    "prnewswire.com": "PR Newswire",
    "globenewswire.com": "GlobeNewswire",
    "apnews.com": "Associated Press",
    "gurufocus.com": "GuruFocus",
    "247wallst.com": "24/7 Wall St.",
    "simplywall.st": "Simply Wall St",
    "kiplinger.com": "Kiplinger",
    "tipranks.com": "TipRanks",
    "fortune.com": "Fortune",
    "marketbeat.com": "MarketBeat",
    "barchart.com": "Barchart",
}


def _publisher_from_link(link: Optional[str]) -> Optional[str]:
    if not link:
        return None
    try:
        host = (urlparse(link).hostname or "").lower().lstrip(".")
    except Exception:
        return None
    if host.startswith("www."):
        host = host[4:]
    if host in _PUBLISHER_DOMAINS:
        return _PUBLISHER_DOMAINS[host]
    parts = host.split(".")
    if len(parts) >= 2:
        apex = ".".join(parts[-2:])
        return _PUBLISHER_DOMAINS.get(apex, apex)
    return host or None


def _entry_publisher(entry) -> Optional[str]:
    """Yahoo rssindex carries publisher in a <source> child; everything else
    we infer from the article URL."""
    src = getattr(entry, "source", None)
    if isinstance(src, dict):
        title = src.get("title")
        if title:
            return title
    elif isinstance(src, str) and src.strip():
        return src.strip()
    return _publisher_from_link(getattr(entry, "link", None))


class YahooRSSAdapter:
    """Pull general financial news feeds and resolve tickers from titles."""

    def __init__(self):
        self.resolver = EntityResolver()

    async def fetch_feed(self, url: str) -> List[Dict]:
        try:
            import feedparser
        except ImportError:
            logger.error("feedparser not installed; RSS adapter inactive")
            return []

        try:
            session = await get_http_session()
            headers = {
                "User-Agent": "Mozilla/5.0 StockOracle/1.0",
                "Accept": "application/rss+xml, application/xml, text/xml",
            }
            async with session.get(
                url, headers=headers, timeout=aiohttp.ClientTimeout(total=30)
            ) as resp:
                if resp.status != 200:
                    logger.warning(f"RSS non-200 from {url}: {resp.status}")
                    return []
                text = await resp.text()
        except Exception as e:
            logger.error(f"RSS fetch error for {url}: {e}")
            return []

        feed = feedparser.parse(text)
        out: List[Dict] = []
        for entry in feed.entries:
            link = getattr(entry, "link", None)
            guid = getattr(entry, "id", None) or link
            title = getattr(entry, "title", "")
            if not guid or not title:
                continue

            published = getattr(entry, "published_parsed", None)
            if published:
                try:
                    pub_dt = datetime(*published[:6], tzinfo=timezone.utc)
                except Exception:
                    pub_dt = datetime.now(timezone.utc)
            else:
                pub_dt = datetime.now(timezone.utc)

            out.append({
                "guid": guid,
                "title": title,
                "publisher": _entry_publisher(entry),
                "published_at": pub_dt,
            })
        return out

    async def collect(self, db: AsyncSession, **_ignored) -> int:
        """Pull all general feeds, dedupe, resolve, persist."""
        await self.resolver.load_aliases(db)

        sem = asyncio.Semaphore(8)

        async def _fetch_one(url: str) -> List[Dict]:
            async with sem:
                return await self.fetch_feed(url)

        results = await asyncio.gather(
            *[_fetch_one(u) for u in _GENERAL_FEEDS], return_exceptions=True
        )

        # Cross-feed deduplication by guid (some headlines syndicate across
        # multiple aggregators).
        seen: Dict[str, Dict] = {}
        for r in results:
            if isinstance(r, Exception) or not isinstance(r, list):
                continue
            for entry in r:
                seen.setdefault(entry["guid"], entry)

        inserted = 0
        for guid, entry in seen.items():
            existing = await db.execute(
                select(OverlayHeadlineEvent.id).where(
                    OverlayHeadlineEvent.article_guid == guid
                )
            )
            if existing.first():
                continue

            matched = self.resolver.resolve_from_title(entry["title"])
            event = OverlayHeadlineEvent(
                article_guid=guid,
                title=entry["title"],
                publisher=entry.get("publisher"),
                published_at=entry["published_at"],
                matched_symbols=matched,
            )
            db.add(event)
            inserted += 1

        if inserted:
            await db.commit()
            logger.info(f"RSS: inserted {inserted} new headline events")

        return inserted