You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
315 lines
10 KiB
Python
315 lines
10 KiB
Python
"""Coverage Check: Validate Stock Oracle data availability for symbol universe.
|
|
|
|
Checks each ticker for:
|
|
1. Price data coverage (daily bars since start_date, require 90%+ trading days)
|
|
2. Filing data (8-K filings, require 3+ filings)
|
|
3. Company info (name, CIK, sector must exist)
|
|
4. Market cap range ($500M - $10B for small/mid-cap)
|
|
|
|
Outputs CSV report + pass/fail summary. Optionally writes a filtered YAML
|
|
with only passing tickers.
|
|
|
|
Usage:
|
|
python -m dev.analysis.coverage_check \
|
|
--symbols-file configs/symbols_smallmid.yaml \
|
|
--start-date 2022-07-01 \
|
|
[--output-dir ./data/analysis] \
|
|
[--write-filtered]
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import csv
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import yaml
|
|
|
|
from libs.oracle_client import FilingsService, FinancialService, PriceService, make_oracle_client
|
|
|
|
|
|
# --- Constants ---
|
|
MIN_MARKET_CAP = 5e8 # $500M
|
|
MAX_MARKET_CAP = 1e10 # $10B
|
|
MIN_FILINGS = 3
|
|
MIN_PRICE_COVERAGE = 0.90 # 90% of expected trading days
|
|
APPROX_TRADING_DAYS_PER_YEAR = 252
|
|
|
|
|
|
def _load_symbols(path: str) -> list[str]:
|
|
with open(path) as f:
|
|
cfg = yaml.safe_load(f) or {}
|
|
symbols = cfg.get("symbols", [])
|
|
# Deduplicate while preserving order
|
|
seen: set[str] = set()
|
|
unique: list[str] = []
|
|
for s in symbols:
|
|
if s not in seen:
|
|
seen.add(s)
|
|
unique.append(s)
|
|
return unique
|
|
|
|
|
|
async def _check_ticker(
|
|
ticker: str,
|
|
price_svc: PriceService,
|
|
filings_svc: FilingsService,
|
|
financial_svc: FinancialService,
|
|
start_date: str,
|
|
expected_trading_days: int,
|
|
) -> dict[str, Any]:
|
|
"""Run all coverage checks for a single ticker."""
|
|
result: dict[str, Any] = {
|
|
"ticker": ticker,
|
|
"price_bars": 0,
|
|
"price_coverage": 0.0,
|
|
"price_pass": False,
|
|
"filing_count": 0,
|
|
"filing_pass": False,
|
|
"has_name": False,
|
|
"has_cik": False,
|
|
"has_sector": False,
|
|
"sector": None,
|
|
"info_pass": False,
|
|
"market_cap": None,
|
|
"market_cap_pass": False,
|
|
"overall_pass": False,
|
|
"error": None,
|
|
}
|
|
|
|
# 1. Price data
|
|
try:
|
|
end_date = datetime.now().strftime("%Y-%m-%d")
|
|
price_resp = await price_svc.get_daily_bars(ticker, start=start_date, end=end_date)
|
|
n_bars = len(price_resp.bars)
|
|
result["price_bars"] = n_bars
|
|
coverage = n_bars / expected_trading_days if expected_trading_days > 0 else 0
|
|
result["price_coverage"] = round(coverage, 3)
|
|
result["price_pass"] = coverage >= MIN_PRICE_COVERAGE
|
|
except Exception as e:
|
|
result["error"] = f"price: {e}"
|
|
|
|
# 2. Filings (8-K)
|
|
try:
|
|
filings_resp = await filings_svc.search_filings(
|
|
ticker, form_type="8-K", start_date=start_date
|
|
)
|
|
n_filings = len(filings_resp.filings)
|
|
result["filing_count"] = n_filings
|
|
result["filing_pass"] = n_filings >= MIN_FILINGS
|
|
except Exception as e:
|
|
err = f"filings: {e}"
|
|
result["error"] = err if not result["error"] else f"{result['error']}; {err}"
|
|
|
|
# 3. Company info + market cap
|
|
try:
|
|
info = await financial_svc.get_company_info(ticker)
|
|
result["has_name"] = bool(info.name)
|
|
result["has_cik"] = bool(info.cik)
|
|
result["has_sector"] = bool(info.sector)
|
|
result["sector"] = info.sector
|
|
result["info_pass"] = all([info.name, info.cik, info.sector])
|
|
if info.market_cap is not None:
|
|
result["market_cap"] = info.market_cap
|
|
result["market_cap_pass"] = MIN_MARKET_CAP <= info.market_cap <= MAX_MARKET_CAP
|
|
else:
|
|
# If market_cap not available, pass this check (can't verify)
|
|
result["market_cap_pass"] = True
|
|
except Exception as e:
|
|
err = f"info: {e}"
|
|
result["error"] = err if not result["error"] else f"{result['error']}; {err}"
|
|
# If company info fails entirely, market_cap check is inconclusive
|
|
result["market_cap_pass"] = True
|
|
|
|
# Overall
|
|
result["overall_pass"] = all([
|
|
result["price_pass"],
|
|
result["filing_pass"],
|
|
result["info_pass"],
|
|
result["market_cap_pass"],
|
|
])
|
|
|
|
return result
|
|
|
|
|
|
async def run_coverage_check(
|
|
symbols: list[str],
|
|
start_date: str,
|
|
output_dir: Path,
|
|
write_filtered: bool = False,
|
|
symbols_file: str | None = None,
|
|
concurrency: int = 5,
|
|
) -> None:
|
|
"""Run coverage check for all symbols."""
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Calculate expected trading days
|
|
start_dt = datetime.strptime(start_date, "%Y-%m-%d")
|
|
years_elapsed = (datetime.now() - start_dt).days / 365.25
|
|
expected_trading_days = int(years_elapsed * APPROX_TRADING_DAYS_PER_YEAR)
|
|
|
|
print(f"\n{'='*80}")
|
|
print(f"Coverage Check — {len(symbols)} tickers, start_date={start_date}")
|
|
print(f"Expected trading days: ~{expected_trading_days}, concurrency={concurrency}")
|
|
print(f"{'='*80}\n")
|
|
|
|
results: list[dict[str, Any]] = []
|
|
sem = asyncio.Semaphore(concurrency)
|
|
counter = {"done": 0}
|
|
|
|
async def _check_with_sem(ticker: str) -> dict[str, Any]:
|
|
async with sem:
|
|
result = await _check_ticker(
|
|
ticker, price_svc, filings_svc, financial_svc,
|
|
start_date, expected_trading_days,
|
|
)
|
|
counter["done"] += 1
|
|
status = "PASS" if result["overall_pass"] else "FAIL"
|
|
mcap_str = f"${result['market_cap']/1e6:.0f}M" if result["market_cap"] else "N/A"
|
|
print(
|
|
f" [{counter['done']:3d}/{len(symbols)}] {ticker:<6s} "
|
|
f"price={result['price_bars']:>4d} bars ({result['price_coverage']:.0%}) "
|
|
f"filings={result['filing_count']:>3d} "
|
|
f"mcap={mcap_str:>8s} "
|
|
f"-> {status}"
|
|
)
|
|
if result["error"]:
|
|
print(f" ! {result['error']}")
|
|
return result
|
|
|
|
async with make_oracle_client() as client:
|
|
price_svc = PriceService(client)
|
|
filings_svc = FilingsService(client)
|
|
financial_svc = FinancialService(client)
|
|
|
|
tasks = [_check_with_sem(ticker) for ticker in symbols]
|
|
results_unordered = await asyncio.gather(*tasks)
|
|
|
|
# Preserve original symbol order
|
|
ticker_to_result = {r["ticker"]: r for r in results_unordered}
|
|
results = [ticker_to_result[s] for s in symbols]
|
|
|
|
# Write CSV
|
|
csv_path = output_dir / "coverage_check.csv"
|
|
if results:
|
|
fieldnames = list(results[0].keys())
|
|
with open(csv_path, "w", newline="") as f:
|
|
writer = csv.DictWriter(f, fieldnames=fieldnames)
|
|
writer.writeheader()
|
|
writer.writerows(results)
|
|
|
|
# Summary
|
|
passed = [r for r in results if r["overall_pass"]]
|
|
failed = [r for r in results if not r["overall_pass"]]
|
|
|
|
print(f"\n{'='*80}")
|
|
print(f"Summary")
|
|
print(f"{'='*80}")
|
|
print(f" Total: {len(results)}")
|
|
print(f" Passed: {len(passed)}")
|
|
print(f" Failed: {len(failed)}")
|
|
|
|
if failed:
|
|
# Breakdown of failure reasons
|
|
price_fail = sum(1 for r in failed if not r["price_pass"])
|
|
filing_fail = sum(1 for r in failed if not r["filing_pass"])
|
|
info_fail = sum(1 for r in failed if not r["info_pass"])
|
|
mcap_fail = sum(1 for r in failed if not r["market_cap_pass"])
|
|
print(f"\n Failure breakdown:")
|
|
print(f" Price coverage < {MIN_PRICE_COVERAGE:.0%}: {price_fail}")
|
|
print(f" Filings < {MIN_FILINGS}: {filing_fail}")
|
|
print(f" Missing company info: {info_fail}")
|
|
print(f" Market cap out of range: {mcap_fail}")
|
|
print(f"\n Failed tickers: {', '.join(r['ticker'] for r in failed)}")
|
|
|
|
# Sector distribution of passed tickers
|
|
sector_counts: dict[str, int] = {}
|
|
for r in passed:
|
|
sector = r.get("sector") or "Unknown"
|
|
sector_counts[sector] = sector_counts.get(sector, 0) + 1
|
|
if sector_counts:
|
|
print(f"\n Sector distribution (passed):")
|
|
for sector, count in sorted(sector_counts.items(), key=lambda x: -x[1]):
|
|
print(f" {sector:<30s} {count:>3d}")
|
|
|
|
print(f"\n CSV report: {csv_path}")
|
|
|
|
# Write filtered YAML
|
|
if write_filtered and passed:
|
|
filtered_path = output_dir / "symbols_smallmid_filtered.yaml"
|
|
filtered_symbols = [r["ticker"] for r in passed]
|
|
with open(filtered_path, "w") as f:
|
|
yaml.dump(
|
|
{"symbols": filtered_symbols},
|
|
f,
|
|
default_flow_style=False,
|
|
sort_keys=False,
|
|
)
|
|
print(f" Filtered YAML ({len(filtered_symbols)} tickers): {filtered_path}")
|
|
if symbols_file:
|
|
print(f"\n To use filtered list:")
|
|
print(f" cp {filtered_path} {symbols_file}")
|
|
|
|
# Go/No-Go
|
|
print(f"\n{'='*80}")
|
|
if len(passed) >= 150:
|
|
print(f"GO: {len(passed)} tickers passed (>= 150 threshold)")
|
|
elif len(passed) >= 100:
|
|
print(f"MARGINAL: {len(passed)} tickers passed (100-150 range, consider adding more)")
|
|
else:
|
|
print(f"NO-GO: Only {len(passed)} tickers passed (< 100, need more tickers)")
|
|
print(f"{'='*80}\n")
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description="Coverage Check for Symbol Universe")
|
|
parser.add_argument(
|
|
"--symbols-file",
|
|
required=True,
|
|
help="Path to symbols YAML file",
|
|
)
|
|
parser.add_argument(
|
|
"--start-date",
|
|
default="2022-07-01",
|
|
help="Start date for data coverage check (YYYY-MM-DD)",
|
|
)
|
|
parser.add_argument(
|
|
"--output-dir",
|
|
default="./data/analysis",
|
|
help="Output directory for CSV report",
|
|
)
|
|
parser.add_argument(
|
|
"--write-filtered",
|
|
action="store_true",
|
|
help="Write filtered YAML with only passing tickers",
|
|
)
|
|
parser.add_argument(
|
|
"--concurrency",
|
|
type=int,
|
|
default=5,
|
|
help="Max concurrent API calls (default: 5)",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
symbols = _load_symbols(args.symbols_file)
|
|
if not symbols:
|
|
print(f"No symbols found in {args.symbols_file}")
|
|
return
|
|
|
|
asyncio.run(
|
|
run_coverage_check(
|
|
symbols=symbols,
|
|
start_date=args.start_date,
|
|
output_dir=Path(args.output_dir),
|
|
write_filtered=args.write_filtered,
|
|
symbols_file=args.symbols_file,
|
|
concurrency=args.concurrency,
|
|
)
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|