stock-oracle/scripts/news_backfill.py

"""
News v2 historical backfill — Finnhub (and future Alpaca catch-up).

Usage (inside the API container):
  python scripts/news_backfill.py \\
      --source finnhub \\
      --tickers AAPL,MSFT,NVDA \\
      --start 2025-04-25 \\
      --end 2026-04-25 \\
      --chunk monthly

  python scripts/news_backfill.py --source finnhub \\
      --tickers-file data/universe/v49_active.txt \\
      --start 2025-04-25 --end 2026-04-25

Estimated runtime (Finnhub free, 60 calls/min, monthly chunks):
  100 tickers × 12 months ≈ 1,200 calls ≈ 20 minutes.

Idempotent: re-runs hit ON CONFLICT DO NOTHING on
(source, source_id, ticker), so overlapping windows are safe.
"""

from __future__ import annotations

import argparse
import asyncio
import logging
import os
import sys
from calendar import monthrange
from datetime import date, datetime, timedelta
from pathlib import Path

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
logger = logging.getLogger("news_backfill")


def _parse_date(s: str) -> date:
    return datetime.strptime(s, "%Y-%m-%d").date()


def _load_tickers(args) -> list[str]:
    tickers: list[str] = []
    if args.tickers:
        tickers.extend(t.strip().upper() for t in args.tickers.split(",") if t.strip())
    if args.tickers_file:
        path = Path(args.tickers_file)
        if not path.exists():
            raise SystemExit(f"--tickers-file not found: {path}")
        for raw in path.read_text().splitlines():
            t = raw.strip().upper()
            if t and not t.startswith("#"):
                tickers.append(t)
    if not tickers:
        raise SystemExit("Provide --tickers and/or --tickers-file")
    # Dedup, preserve order
    seen: set[str] = set()
    out: list[str] = []
    for t in tickers:
        if t not in seen:
            seen.add(t)
            out.append(t)
    return out


def _monthly_chunks(start: date, end: date) -> list[tuple[date, date]]:
    """Split [start, end] into per-month [chunk_start, chunk_end] inclusive bounds."""
    chunks: list[tuple[date, date]] = []
    cur = date(start.year, start.month, 1)
    if cur < start:
        cur = start
    while cur <= end:
        last_day_of_month = monthrange(cur.year, cur.month)[1]
        chunk_end = min(end, date(cur.year, cur.month, last_day_of_month))
        chunks.append((cur, chunk_end))
        # Advance to first of next month
        if cur.month == 12:
            cur = date(cur.year + 1, 1, 1)
        else:
            cur = date(cur.year, cur.month + 1, 1)
    return chunks


def _daily_chunks(start: date, end: date) -> list[tuple[date, date]]:
    return [(start + timedelta(days=i), start + timedelta(days=i)) for i in range((end - start).days + 1)]


async def _backfill_finnhub(tickers: list[str], start: date, end: date, chunk: str) -> None:
    from app.services.news.finnhub_client import FinnhubClient
    from app.services.news.headline_ingest_service import (
        finnhub_articles_to_rows,
        insert_headline_rows,
    )

    client = FinnhubClient()
    if not client.is_configured():
        raise SystemExit("FINNHUB_API_KEY not set in environment")

    if chunk == "monthly":
        ranges = _monthly_chunks(start, end)
    elif chunk == "daily":
        ranges = _daily_chunks(start, end)
    else:
        raise SystemExit(f"Unknown --chunk: {chunk}")

    total_calls = len(tickers) * len(ranges)
    logger.info(
        f"Finnhub backfill: {len(tickers)} tickers × {len(ranges)} {chunk} chunks "
        f"= {total_calls} calls (~{total_calls / 60:.1f} min @ 60 cpm)"
    )

    grand_inserted = 0
    grand_calls = 0
    try:
        for ticker in tickers:
            ticker_inserted = 0
            for (chunk_start, chunk_end) in ranges:
                try:
                    payload = await client.fetch_company_news(ticker, chunk_start, chunk_end)
                except Exception as e:
                    logger.error(f"Finnhub fetch failed {ticker} {chunk_start}..{chunk_end}: {e}")
                    continue
                grand_calls += 1
                if not payload:
                    continue
                rows = finnhub_articles_to_rows(ticker, payload)
                if not rows:
                    continue
                inserted = await insert_headline_rows(rows)
                ticker_inserted += inserted
                grand_inserted += inserted
            logger.info(f"  {ticker}: +{ticker_inserted} new (cumulative {grand_inserted}, {grand_calls}/{total_calls} calls)")
    finally:
        await client.close()

    logger.info(f"Finnhub backfill done — {grand_inserted} new headlines inserted in {grand_calls} calls")


def main() -> None:
    parser = argparse.ArgumentParser(description="News v2 historical backfill")
    parser.add_argument("--source", required=True, choices=["finnhub"], help="Vendor source")
    parser.add_argument("--tickers", help="Comma-separated ticker list")
    parser.add_argument("--tickers-file", help="File with one ticker per line (# comments OK)")
    parser.add_argument("--start", required=True, type=_parse_date, help="YYYY-MM-DD inclusive")
    parser.add_argument("--end", required=True, type=_parse_date, help="YYYY-MM-DD inclusive")
    parser.add_argument("--chunk", default="monthly", choices=["monthly", "daily"], help="Per-call window size")

    args = parser.parse_args()

    if args.start > args.end:
        raise SystemExit("--start must be <= --end")

    tickers = _load_tickers(args)
    logger.info(f"Loaded {len(tickers)} tickers")

    if args.source == "finnhub":
        asyncio.run(_backfill_finnhub(tickers, args.start, args.end, args.chunk))


if __name__ == "__main__":
    main()