fithia2/apps/tools/universe_builder.py

"""Universe Builder: expand symbol universe via Stock Oracle screener API.

Fetches large-cap liquid equities from screener, merges with existing
symbols.yaml, and writes the union back.

Usage:
    python -m apps.tools.universe_builder [--keep-existing] [--output configs/symbols.yaml]
"""
from __future__ import annotations

import argparse
import asyncio
from pathlib import Path

import yaml

from libs.common.config import get_settings
from libs.common.logging import configure_logging, get_logger
from libs.oracle_client import ScreenerService, make_oracle_client

logger = get_logger(__name__)

# ── Screener defaults ──────────────────────────────────────────────
SCREENER_DEFAULTS = {
    "market_cap_min": 10_000_000_000,   # $10B
    "min_avg_volume": 1_000_000,        # 1M shares/day
    "exchange": "NYSE,NASDAQ",
    "exclude_types": "ETF,FUND",
    "price_min": 5,
}


def load_existing_symbols(path: Path) -> list[str]:
    """Load current symbols.yaml and return list of tickers."""
    if not path.exists():
        return []
    with open(path) as f:
        cfg = yaml.safe_load(f) or {}
    return cfg.get("symbols", [])


def build_symbols_yaml(
    existing: list[str],
    screener_results: list[dict],
) -> str:
    """Build YAML content: existing-only tickers first, then screener (sorted)."""
    screener_set: set[str] = set()
    for item in screener_results:
        ticker = item.get("symbol", "")
        if ticker:
            screener_set.add(ticker.upper())

    existing_set = {s.upper() for s in existing}
    existing_only = sorted(existing_set - screener_set)
    screener_sorted = sorted(screener_set)

    lines = ["symbols:"]
    if existing_only:
        lines.append("  # --- Existing-only (not in screener, kept) ---")
        for t in existing_only:
            lines.append(f"  - {t}")
    lines.append(f"  # --- Screener ($10B+ mcap, 1M+ vol) — {len(screener_sorted)} symbols ---")
    for t in screener_sorted:
        lines.append(f"  - {t}")
    lines.append("")  # trailing newline

    return "\n".join(lines)


async def run_universe_builder(
    keep_existing: bool,
    output_path: Path,
) -> dict[str, int]:
    existing = load_existing_symbols(output_path) if keep_existing else []
    existing_set = {s.upper() for s in existing}

    async with make_oracle_client() as client:
        svc = ScreenerService(client)
        screener_stocks = await svc.search_all_stocks(**SCREENER_DEFAULTS)

    screener_results = [{"symbol": s.symbol, "name": s.name} for s in screener_stocks]
    screener_tickers = {s.symbol.upper() for s in screener_stocks if s.symbol}

    overlap = existing_set & screener_tickers
    new_only = screener_tickers - existing_set
    existing_only = existing_set - screener_tickers
    union = existing_set | screener_tickers

    stats = {
        "screener": len(screener_tickers),
        "existing": len(existing_set),
        "overlap": len(overlap),
        "existing_only": len(existing_only),
        "new": len(new_only),
        "total": len(union),
    }

    yaml_content = build_symbols_yaml(existing, screener_results)
    output_path.parent.mkdir(parents=True, exist_ok=True)
    output_path.write_text(yaml_content)

    # Report
    logger.info("universe_report", **stats)
    print(f"\n{'='*50}")
    print("Universe Builder Report")
    print(f"{'='*50}")
    print(f"  Screener results:      {stats['screener']:>6}")
    print(f"  Existing symbols:      {stats['existing']:>6}")
    print(f"  Overlap:               {stats['overlap']:>6}")
    print(f"  Existing-only (kept):  {stats['existing_only']:>6}")
    print(f"  New additions:         {stats['new']:>6}")
    print(f"  Total (union):         {stats['total']:>6}")
    print(f"{'='*50}")
    print(f"  Written to: {output_path}")

    if existing_only:
        print(f"\n  Existing-only tickers (kept): {sorted(existing_only)}")

    return stats


def main() -> None:
    parser = argparse.ArgumentParser(
        description="Build symbol universe from Stock Oracle screener",
    )
    parser.add_argument(
        "--keep-existing",
        action="store_true",
        default=True,
        help="Keep all existing symbols (default: True)",
    )
    parser.add_argument(
        "--no-keep-existing",
        action="store_false",
        dest="keep_existing",
        help="Replace existing symbols entirely with screener results",
    )
    parser.add_argument(
        "--output",
        default="configs/symbols.yaml",
        help="Output YAML path (default: configs/symbols.yaml)",
    )
    args = parser.parse_args()

    settings = get_settings()
    configure_logging(settings.log_level)

    asyncio.run(run_universe_builder(
        keep_existing=args.keep_existing,
        output_path=Path(args.output),
    ))


if __name__ == "__main__":
    main()