|
|
"""
|
|
|
FINRA RegSHO daily short sale volume data service
|
|
|
"""
|
|
|
|
|
|
import logging
|
|
|
from datetime import date, datetime, timedelta, timezone
|
|
|
from typing import Dict, List, Optional, Tuple
|
|
|
|
|
|
import aiohttp
|
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
from sqlalchemy import select, and_, func, desc
|
|
|
|
|
|
from app.models.finra_short_volume import FinraShortVolume
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
FINRA_CDN_BASE = "https://cdn.finra.org/equity/regsho/daily"
|
|
|
|
|
|
|
|
|
class FinraShortVolumeService:
|
|
|
"""FINRA RegSHO daily short sale volume data"""
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
# Fetch & Parse
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
|
async def fetch_daily_file(self, target_date: date) -> Optional[str]:
|
|
|
"""
|
|
|
Download the CNMS short volume file for *target_date*.
|
|
|
|
|
|
Returns the raw text or None if 404 (weekend/holiday).
|
|
|
"""
|
|
|
filename = f"CNMSshvol{target_date:%Y%m%d}.txt"
|
|
|
url = f"{FINRA_CDN_BASE}/{filename}"
|
|
|
|
|
|
try:
|
|
|
async with aiohttp.ClientSession() as session:
|
|
|
async with session.get(url, timeout=aiohttp.ClientTimeout(total=30)) as resp:
|
|
|
if resp.status == 404:
|
|
|
logger.debug(f"FINRA file not found (holiday/weekend): {filename}")
|
|
|
return None
|
|
|
resp.raise_for_status()
|
|
|
text = await resp.text()
|
|
|
logger.info(f"FINRA: downloaded {filename} ({len(text)} bytes)")
|
|
|
return text
|
|
|
except aiohttp.ClientError as e:
|
|
|
logger.error(f"FINRA download error for {filename}: {e}")
|
|
|
return None
|
|
|
|
|
|
def parse_short_volume_file(self, text: str) -> List[Dict]:
|
|
|
"""
|
|
|
Parse pipe-delimited FINRA short volume text.
|
|
|
|
|
|
Expected header:
|
|
|
Date|Symbol|ShortVolume|ShortExemptVolume|TotalVolume|Market
|
|
|
|
|
|
Returns list of dicts ready for DB insertion.
|
|
|
"""
|
|
|
lines = text.strip().splitlines()
|
|
|
if len(lines) < 2:
|
|
|
return []
|
|
|
|
|
|
records: List[Dict] = []
|
|
|
for line in lines[1:]:
|
|
|
parts = line.split("|")
|
|
|
if len(parts) < 6:
|
|
|
continue
|
|
|
raw_date, symbol, short_vol, exempt_vol, total_vol, market = (
|
|
|
parts[0].strip(),
|
|
|
parts[1].strip(),
|
|
|
parts[2].strip(),
|
|
|
parts[3].strip(),
|
|
|
parts[4].strip(),
|
|
|
parts[5].strip(),
|
|
|
)
|
|
|
if not symbol or not total_vol:
|
|
|
continue
|
|
|
|
|
|
try:
|
|
|
dt = datetime.strptime(raw_date, "%Y%m%d").replace(tzinfo=timezone.utc)
|
|
|
sv = float(short_vol)
|
|
|
sev = float(exempt_vol) if exempt_vol else 0.0
|
|
|
tv = float(total_vol)
|
|
|
ratio = sv / tv if tv > 0 else 0.0
|
|
|
except (ValueError, ZeroDivisionError):
|
|
|
continue
|
|
|
|
|
|
records.append(
|
|
|
{
|
|
|
"date": dt,
|
|
|
"symbol": symbol.upper(),
|
|
|
"short_volume": sv,
|
|
|
"short_exempt_volume": sev,
|
|
|
"total_volume": tv,
|
|
|
"market": market if market else None,
|
|
|
"short_ratio": round(ratio, 6),
|
|
|
}
|
|
|
)
|
|
|
|
|
|
return records
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
# Ingest
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
|
async def ingest_date(
|
|
|
self, db: AsyncSession, target_date: date, force_refresh: bool = False
|
|
|
) -> int:
|
|
|
"""
|
|
|
Download, parse, and bulk-upsert data for a single date.
|
|
|
|
|
|
Returns number of records inserted.
|
|
|
"""
|
|
|
if not force_refresh:
|
|
|
# Check if already ingested
|
|
|
count = await self._count_for_date(db, target_date)
|
|
|
if count > 0:
|
|
|
logger.info(f"FINRA date {target_date} already ingested ({count} rows)")
|
|
|
return 0
|
|
|
|
|
|
text = await self.fetch_daily_file(target_date)
|
|
|
if text is None:
|
|
|
return 0
|
|
|
|
|
|
records = self.parse_short_volume_file(text)
|
|
|
if not records:
|
|
|
return 0
|
|
|
|
|
|
# Bulk insert – skip duplicates
|
|
|
inserted = 0
|
|
|
for rec in records:
|
|
|
existing = await db.execute(
|
|
|
select(FinraShortVolume.id).where(
|
|
|
and_(
|
|
|
FinraShortVolume.symbol == rec["symbol"],
|
|
|
FinraShortVolume.date == rec["date"],
|
|
|
FinraShortVolume.market == rec["market"],
|
|
|
)
|
|
|
)
|
|
|
)
|
|
|
if existing.first():
|
|
|
continue
|
|
|
|
|
|
db.add(FinraShortVolume(**rec))
|
|
|
inserted += 1
|
|
|
|
|
|
if inserted:
|
|
|
await db.commit()
|
|
|
logger.info(f"FINRA: ingested {inserted} records for {target_date}")
|
|
|
|
|
|
return inserted
|
|
|
|
|
|
async def ingest_date_range(
|
|
|
self,
|
|
|
db: AsyncSession,
|
|
|
start_date: date,
|
|
|
end_date: date,
|
|
|
force_refresh: bool = False,
|
|
|
) -> int:
|
|
|
"""Ingest short volume data for a date range (weekdays only)."""
|
|
|
total = 0
|
|
|
current = start_date
|
|
|
while current <= end_date:
|
|
|
# Skip weekends
|
|
|
if current.weekday() < 5:
|
|
|
count = await self.ingest_date(db, current, force_refresh=force_refresh)
|
|
|
total += count
|
|
|
current += timedelta(days=1)
|
|
|
|
|
|
logger.info(f"FINRA range ingest: {total} records from {start_date} to {end_date}")
|
|
|
return total
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
# Query
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
|
async def get_short_volume(
|
|
|
self,
|
|
|
db: AsyncSession,
|
|
|
symbol: str,
|
|
|
start_date: Optional[date] = None,
|
|
|
end_date: Optional[date] = None,
|
|
|
limit: int = 100,
|
|
|
) -> Tuple[List[FinraShortVolume], int]:
|
|
|
"""
|
|
|
Query short volume for a symbol within a date range.
|
|
|
|
|
|
If no data found, attempts auto-ingest for the requested range.
|
|
|
"""
|
|
|
symbol = symbol.upper()
|
|
|
|
|
|
conditions = [FinraShortVolume.symbol == symbol]
|
|
|
if start_date:
|
|
|
conditions.append(
|
|
|
FinraShortVolume.date >= datetime.combine(start_date, datetime.min.time()).replace(tzinfo=timezone.utc)
|
|
|
)
|
|
|
if end_date:
|
|
|
conditions.append(
|
|
|
FinraShortVolume.date <= datetime.combine(end_date, datetime.min.time()).replace(tzinfo=timezone.utc)
|
|
|
)
|
|
|
|
|
|
# Count
|
|
|
count_q = await db.execute(
|
|
|
select(func.count(FinraShortVolume.id)).where(and_(*conditions))
|
|
|
)
|
|
|
total_count = count_q.scalar() or 0
|
|
|
|
|
|
# If no data, try auto-ingest
|
|
|
if total_count == 0 and start_date and end_date:
|
|
|
ingested = await self.ingest_date_range(db, start_date, end_date)
|
|
|
if ingested > 0:
|
|
|
count_q = await db.execute(
|
|
|
select(func.count(FinraShortVolume.id)).where(and_(*conditions))
|
|
|
)
|
|
|
total_count = count_q.scalar() or 0
|
|
|
|
|
|
# Fetch
|
|
|
result = await db.execute(
|
|
|
select(FinraShortVolume)
|
|
|
.where(and_(*conditions))
|
|
|
.order_by(desc(FinraShortVolume.date))
|
|
|
.limit(limit)
|
|
|
)
|
|
|
rows = result.scalars().all()
|
|
|
|
|
|
return rows, total_count
|
|
|
|
|
|
async def get_short_ratio_history(
|
|
|
self,
|
|
|
db: AsyncSession,
|
|
|
symbol: str,
|
|
|
days: int = 30,
|
|
|
) -> List[Dict]:
|
|
|
"""
|
|
|
Return short_ratio history for the last *days* trading days.
|
|
|
|
|
|
Groups by date (aggregates across markets).
|
|
|
"""
|
|
|
symbol = symbol.upper()
|
|
|
cutoff = datetime.now(timezone.utc) - timedelta(days=days)
|
|
|
|
|
|
result = await db.execute(
|
|
|
select(
|
|
|
FinraShortVolume.date,
|
|
|
func.sum(FinraShortVolume.short_volume).label("short_volume"),
|
|
|
func.sum(FinraShortVolume.short_exempt_volume).label("short_exempt_volume"),
|
|
|
func.sum(FinraShortVolume.total_volume).label("total_volume"),
|
|
|
)
|
|
|
.where(
|
|
|
and_(
|
|
|
FinraShortVolume.symbol == symbol,
|
|
|
FinraShortVolume.date >= cutoff,
|
|
|
)
|
|
|
)
|
|
|
.group_by(FinraShortVolume.date)
|
|
|
.order_by(FinraShortVolume.date)
|
|
|
)
|
|
|
|
|
|
history = []
|
|
|
for row in result.fetchall():
|
|
|
dt, sv, sev, tv = row
|
|
|
ratio = sv / tv if tv and tv > 0 else 0.0
|
|
|
history.append(
|
|
|
{
|
|
|
"date": dt.date() if isinstance(dt, datetime) else dt,
|
|
|
"short_volume": sv,
|
|
|
"short_exempt_volume": sev,
|
|
|
"total_volume": tv,
|
|
|
"short_ratio": round(ratio, 6),
|
|
|
}
|
|
|
)
|
|
|
|
|
|
# Auto-ingest if empty
|
|
|
if not history:
|
|
|
start = (datetime.now(timezone.utc) - timedelta(days=days)).date()
|
|
|
end = datetime.now(timezone.utc).date()
|
|
|
ingested = await self.ingest_date_range(db, start, end)
|
|
|
if ingested > 0:
|
|
|
return await self.get_short_ratio_history(db, symbol, days)
|
|
|
|
|
|
return history
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
# Helpers
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
|
async def _count_for_date(self, db: AsyncSession, target_date: date) -> int:
|
|
|
dt = datetime.combine(target_date, datetime.min.time()).replace(tzinfo=timezone.utc)
|
|
|
result = await db.execute(
|
|
|
select(func.count(FinraShortVolume.id)).where(FinraShortVolume.date == dt)
|
|
|
)
|
|
|
return result.scalar() or 0
|