fithia2/apps/intraday_bt/scripts/build_daily_from_intraday.py

"""Synthesize daily OHLCV bars from intraday 5-min cache.

Walks data/cache/intraday/{TICKER}/*.parquet and aggregates each day's
market-hours bars (09:30 ≤ t < 16:00 ET) into a single daily row.

Output: data/cache/daily_synth_full/{TICKER}.parquet — same schema as the
existing snapshots (date, open, high, low, close, volume).

Usage:
    python -m apps.intraday_bt.scripts.build_daily_from_intraday \
        --output-dir data/cache/daily_synth_full \
        --tickers ALL                                  # or comma-list

Skips days with fewer than --min-bars (default 50) market-hours bars.
"""
from __future__ import annotations

import argparse
import datetime as dt
import time
from pathlib import Path
from zoneinfo import ZoneInfo

import pandas as pd

_ET = ZoneInfo("America/New_York")
_INTRADAY_DIR = Path("data/cache/intraday")
_DEFAULT_OUT = Path("data/cache/daily_synth_full")


def _aggregate_one_day(df: pd.DataFrame, min_bars: int = 50) -> dict | None:
    if df.empty:
        return None
    ts = pd.to_datetime(df["timestamp"])
    if ts.iloc[0].tzinfo is None:
        ts_et = ts.dt.tz_localize("UTC").dt.tz_convert(_ET)
    else:
        ts_et = ts.dt.tz_convert(_ET)
    mask = (ts_et.dt.time >= dt.time(9, 30)) & (ts_et.dt.time < dt.time(16, 0))
    m = df[mask.values]
    if len(m) < min_bars:
        return None
    et_dates = ts_et[mask.values]
    return {
        "date": str(et_dates.iloc[0].date()),
        "open": float(m.iloc[0]["open"]),
        "high": float(m["high"].max()),
        "low": float(m["low"].min()),
        "close": float(m.iloc[-1]["close"]),
        "volume": float(m["volume"].sum()),
    }


def _build_ticker(
    ticker: str,
    out_dir: Path,
    min_bars: int = 50,
    year_filter: tuple[int, int] | None = None,
) -> tuple[int, int]:
    tdir = _INTRADAY_DIR / ticker
    if not tdir.is_dir():
        return (0, 0)
    rows: list[dict] = []
    skipped = 0
    files = sorted(tdir.glob("*.parquet"))
    if year_filter is not None:
        lo, hi = year_filter
        files = [f for f in files if lo <= int(f.name[:4]) <= hi]
    for f in files:
        try:
            df = pd.read_parquet(f, columns=["timestamp", "open", "high", "low", "close", "volume"])
        except Exception:
            skipped += 1
            continue
        agg = _aggregate_one_day(df, min_bars=min_bars)
        if agg is None:
            skipped += 1
            continue
        rows.append(agg)
    if not rows:
        return (0, skipped)
    out_df = pd.DataFrame(rows).sort_values("date").drop_duplicates("date", keep="last")
    out_path = out_dir / f"{ticker}.parquet"
    out_df.to_parquet(out_path, index=False)
    return (len(out_df), skipped)


def _worker(args: tuple) -> tuple[str, int, int]:
    ticker, out_dir_str, min_bars, year_filter = args
    rows, skipped = _build_ticker(ticker, Path(out_dir_str), min_bars, year_filter)
    return (ticker, rows, skipped)


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--output-dir", default=str(_DEFAULT_OUT))
    ap.add_argument("--tickers", default="ALL",
                    help="Comma-separated ticker list, or ALL for everything in intraday cache.")
    ap.add_argument("--min-bars", type=int, default=50,
                    help="Minimum market-hours bars to count a day (default 50).")
    ap.add_argument("--limit", type=int, default=None,
                    help="Optional: process at most N tickers (for testing).")
    ap.add_argument("--year-from", type=int, default=None,
                    help="Restrict to files with year >= this (e.g., 2022).")
    ap.add_argument("--year-to", type=int, default=None,
                    help="Restrict to files with year <= this (e.g., 2024).")
    ap.add_argument("--workers", type=int, default=8,
                    help="Parallel workers (default 8).")
    args = ap.parse_args()

    out_dir = Path(args.output_dir)
    out_dir.mkdir(parents=True, exist_ok=True)

    if args.tickers.upper() == "ALL":
        tickers = sorted(d.name for d in _INTRADAY_DIR.iterdir() if d.is_dir())
    else:
        tickers = [t.strip().upper() for t in args.tickers.split(",") if t.strip()]
    if args.limit:
        tickers = tickers[: args.limit]

    yf = None
    if args.year_from is not None or args.year_to is not None:
        yf = (args.year_from or 1900, args.year_to or 9999)

    print(f"Synthesizing daily bars for {len(tickers)} tickers → {out_dir} "
          f"(workers={args.workers}, year_filter={yf})")
    t0 = time.time()
    total_rows = 0
    nonempty = 0

    if args.workers <= 1:
        for i, t in enumerate(tickers):
            rows, _ = _build_ticker(t, out_dir, min_bars=args.min_bars, year_filter=yf)
            total_rows += rows
            if rows > 0:
                nonempty += 1
            if (i + 1) % 200 == 0 or i + 1 == len(tickers):
                elapsed = time.time() - t0
                rate = (i + 1) / elapsed if elapsed > 0 else 0
                print(f"  [{i+1}/{len(tickers)}] elapsed {elapsed:.1f}s, "
                      f"{rate:.1f} tk/s, {nonempty} nonempty, {total_rows} rows")
    else:
        from concurrent.futures import ProcessPoolExecutor, as_completed
        tasks = [(t, str(out_dir), args.min_bars, yf) for t in tickers]
        completed = 0
        with ProcessPoolExecutor(max_workers=args.workers) as ex:
            futures = {ex.submit(_worker, task): task[0] for task in tasks}
            for fut in as_completed(futures):
                ticker, rows, _ = fut.result()
                completed += 1
                total_rows += rows
                if rows > 0:
                    nonempty += 1
                if completed % 200 == 0 or completed == len(tickers):
                    elapsed = time.time() - t0
                    rate = completed / elapsed if elapsed > 0 else 0
                    print(f"  [{completed}/{len(tickers)}] elapsed {elapsed:.1f}s, "
                          f"{rate:.1f} tk/s, {nonempty} nonempty, {total_rows} rows")

    print(f"\nDone. {nonempty}/{len(tickers)} tickers had at least one valid day. "
          f"{total_rows} total rows. {time.time()-t0:.1f}s")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())