|
|
"""
|
|
|
Phase 0: Audit FINRA short_sale_daily coverage for V25 short-volume overlay.
|
|
|
|
|
|
Checks whether the DB has sufficient historical short-volume data for:
|
|
|
- The midlarge universe over the 200d backtest window (2025-07-07 → 2026-04-21)
|
|
|
- The V24 200d trade set (ticker, entry_date) pairs
|
|
|
|
|
|
Gates:
|
|
|
G0a: Median universe coverage ≥ 80%
|
|
|
G0b: V24 trade-set coverage ≥ 80% (≥ 114/142 trades)
|
|
|
"""
|
|
|
from __future__ import annotations
|
|
|
|
|
|
import asyncio
|
|
|
import datetime as dt
|
|
|
import json
|
|
|
import os
|
|
|
import sys
|
|
|
from pathlib import Path
|
|
|
|
|
|
import asyncpg
|
|
|
import yaml
|
|
|
|
|
|
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
|
|
|
|
|
from libs.common.config import get_settings
|
|
|
from libs.common.time_utils import trading_days_between
|
|
|
|
|
|
UNIVERSE_FILE = "configs/symbols_midlarge_snapshot_exact.yaml"
|
|
|
V24_RUN_FILE = "runs/intraday_orb/intraday_20260421_205350_67d5361a.json"
|
|
|
LOOKBACK_DAYS = 200
|
|
|
END_DATE = dt.date(2026, 4, 21)
|
|
|
|
|
|
|
|
|
async def main() -> None:
|
|
|
print("=== Phase 0: short_sale_daily Coverage Audit ===\n")
|
|
|
|
|
|
# 1. Compute 200d window
|
|
|
all_td = trading_days_between(END_DATE - dt.timedelta(days=400), END_DATE)
|
|
|
trading_days = [d.isoformat() for d in all_td[-LOOKBACK_DAYS:]]
|
|
|
start_date = trading_days[0]
|
|
|
end_date = trading_days[-1]
|
|
|
print(f"Window: {start_date} → {end_date} ({len(trading_days)} trading days)")
|
|
|
|
|
|
# 2. Load universe
|
|
|
with open(UNIVERSE_FILE) as f:
|
|
|
udata = yaml.safe_load(f)
|
|
|
universe: list[str] = udata.get("symbols", udata) if isinstance(udata, dict) else udata
|
|
|
print(f"Universe: {len(universe)} tickers")
|
|
|
|
|
|
# 3. Connect to DB (raw asyncpg, same pattern as enrich_tier2_features.py:263)
|
|
|
dsn = get_settings().postgres_dsn.replace("+asyncpg", "")
|
|
|
conn = await asyncpg.connect(dsn=dsn)
|
|
|
|
|
|
try:
|
|
|
# 4. Check overall DB date range for short_sale_daily
|
|
|
date_range_row = await conn.fetchrow(
|
|
|
"SELECT MIN(trade_date)::text AS min_date, MAX(trade_date)::text AS max_date, COUNT(*) AS total_rows FROM short_sale_daily"
|
|
|
)
|
|
|
print(f"\nDB table short_sale_daily:")
|
|
|
print(f" Total rows: {date_range_row['total_rows']:,}")
|
|
|
print(f" Date range: {date_range_row['min_date']} → {date_range_row['max_date']}")
|
|
|
|
|
|
# 5. Bulk fetch coverage for universe tickers × window
|
|
|
rows = await conn.fetch(
|
|
|
"""
|
|
|
SELECT ticker_raw,
|
|
|
COUNT(DISTINCT trade_date) AS row_count,
|
|
|
MIN(trade_date)::text AS min_date,
|
|
|
MAX(trade_date)::text AS max_date
|
|
|
FROM short_sale_daily
|
|
|
WHERE ticker_raw = ANY($1)
|
|
|
AND trade_date >= $2::date
|
|
|
AND trade_date <= $3::date
|
|
|
AND total_volume IS NOT NULL
|
|
|
AND total_volume > 0
|
|
|
GROUP BY ticker_raw
|
|
|
""",
|
|
|
universe, dt.date.fromisoformat(start_date), dt.date.fromisoformat(end_date),
|
|
|
)
|
|
|
|
|
|
coverage_by_ticker: dict[str, int] = {r["ticker_raw"]: r["row_count"] for r in rows}
|
|
|
n_trading = len(trading_days)
|
|
|
|
|
|
pcts = []
|
|
|
zero_tickers = []
|
|
|
for ticker in universe:
|
|
|
count = coverage_by_ticker.get(ticker, 0)
|
|
|
pct = count / n_trading
|
|
|
pcts.append(pct)
|
|
|
if count == 0:
|
|
|
zero_tickers.append(ticker)
|
|
|
|
|
|
pcts_sorted = sorted(pcts)
|
|
|
median_pct = pcts_sorted[len(pcts_sorted) // 2]
|
|
|
mean_pct = sum(pcts) / len(pcts)
|
|
|
tickers_above_80 = sum(1 for p in pcts if p >= 0.80)
|
|
|
tickers_above_60 = sum(1 for p in pcts if p >= 0.60)
|
|
|
|
|
|
print(f"\nUniverse coverage over window:")
|
|
|
print(f" Median: {median_pct*100:.1f}%")
|
|
|
print(f" Mean: {mean_pct*100:.1f}%")
|
|
|
print(f" Tickers ≥80% coverage: {tickers_above_80}/{len(universe)}")
|
|
|
print(f" Tickers ≥60% coverage: {tickers_above_60}/{len(universe)}")
|
|
|
print(f" Zero-coverage tickers: {len(zero_tickers)}")
|
|
|
if zero_tickers[:10]:
|
|
|
print(f" (first 10): {zero_tickers[:10]}")
|
|
|
|
|
|
gate_g0a = median_pct >= 0.80
|
|
|
print(f"\n G0a Median ≥ 80%: {median_pct*100:.1f}% → {'PASS ✓' if gate_g0a else 'FAIL ✗'}")
|
|
|
|
|
|
# 6. V24 trade-set coverage
|
|
|
if Path(V24_RUN_FILE).exists():
|
|
|
with open(V24_RUN_FILE) as f:
|
|
|
v24_data = json.load(f)
|
|
|
v24_trades = v24_data.get("trades", [])
|
|
|
print(f"\nV24 trade set: {len(v24_trades)} trades")
|
|
|
|
|
|
# For each trade, check if short volume exists for the entry date and prev 20 days
|
|
|
# Use a single bulk query for all (ticker, date) combos within window
|
|
|
trade_tickers = list({t["ticker"] for t in v24_trades})
|
|
|
short_rows = await conn.fetch(
|
|
|
"""
|
|
|
SELECT ticker_raw, trade_date::text AS date
|
|
|
FROM short_sale_daily
|
|
|
WHERE ticker_raw = ANY($1)
|
|
|
AND trade_date >= $2::date
|
|
|
AND trade_date <= $3::date
|
|
|
AND total_volume IS NOT NULL
|
|
|
AND total_volume > 0
|
|
|
""",
|
|
|
trade_tickers, dt.date.fromisoformat(start_date), dt.date.fromisoformat(end_date),
|
|
|
)
|
|
|
|
|
|
# Build set of (ticker, date) with data
|
|
|
short_set: set[tuple[str, str]] = {(r["ticker_raw"], r["date"]) for r in short_rows}
|
|
|
|
|
|
# For each trade, check if the prior-day short volume is available
|
|
|
# (any short data in the 10 trading days before entry = sufficient for prior-day feature)
|
|
|
trade_idx = {d: i for i, d in enumerate(trading_days)}
|
|
|
covered = 0
|
|
|
missing_trades: list[str] = []
|
|
|
for trade in v24_trades:
|
|
|
ticker = trade["ticker"]
|
|
|
entry_date = trade["date"]
|
|
|
# Find prior trading days (look back up to 10)
|
|
|
if entry_date in trade_idx:
|
|
|
idx = trade_idx[entry_date]
|
|
|
prior_days = trading_days[max(0, idx - 10):idx]
|
|
|
else:
|
|
|
prior_days = []
|
|
|
has_prior = any((ticker, d) in short_set for d in prior_days)
|
|
|
if has_prior:
|
|
|
covered += 1
|
|
|
else:
|
|
|
missing_trades.append(f"{trade['date']}:{ticker}")
|
|
|
|
|
|
trade_coverage_pct = covered / len(v24_trades)
|
|
|
gate_g0b = covered >= int(0.80 * len(v24_trades))
|
|
|
print(f" Trades with prior-day short data: {covered}/{len(v24_trades)} ({trade_coverage_pct*100:.1f}%)")
|
|
|
print(f" G0b V24 trade coverage ≥ 80%: {trade_coverage_pct*100:.1f}% → {'PASS ✓' if gate_g0b else 'FAIL ✗'}")
|
|
|
if missing_trades[:10]:
|
|
|
print(f" Missing trades (first 10): {missing_trades[:10]}")
|
|
|
|
|
|
both_pass = gate_g0a and gate_g0b
|
|
|
else:
|
|
|
print(f"\nWARNING: V24 run file not found: {V24_RUN_FILE}")
|
|
|
both_pass = gate_g0a
|
|
|
|
|
|
print(f"\n{'='*60}")
|
|
|
print(f"PHASE 0 VERDICT: {'PASS — proceed to Phase 1' if both_pass else 'FAIL — backfill required before Phase 1'}")
|
|
|
if not both_pass:
|
|
|
print(" Action: run `apps/sync/short_volume_sync/main.py --days 300` then re-audit")
|
|
|
|
|
|
finally:
|
|
|
await conn.close()
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
asyncio.run(main())
|