You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
84 lines
2.7 KiB
Python
84 lines
2.7 KiB
Python
"""
|
|
FINRA short-volume 2020 갭 메우기 (2020-04-01 ~ 2020-10-31).
|
|
|
|
DB 조사 결과: 전 구간(2018-08 ~ 2026-05)에서 유일한 데이터 결손이
|
|
2020-04-01 ~ 2020-10-31 (약 138 평일). COVID 폭락장 + 회복장을 포함하는
|
|
이 구간이 없으면 멀티레짐 검정력이 크게 훼손된다.
|
|
|
|
특성:
|
|
- FINRA CDN 공개 무료 (API 키 불필요)
|
|
- idempotent: 이미 수집된 날짜는 자동 스킵 (_count_for_date 검사)
|
|
- 주말/공휴일(파일 없는 날)은 404 수신 후 자동 스킵
|
|
- 1년치 약 20-40분 소요 → 이 스크립트는 7개월, 약 10-20분 예상
|
|
|
|
Usage (컨테이너 내부):
|
|
python scripts/backfill_finra_2020_gap.py
|
|
|
|
Usage (호스트):
|
|
docker exec stock_oracle_api python scripts/backfill_finra_2020_gap.py
|
|
"""
|
|
|
|
import asyncio
|
|
import fcntl
|
|
import logging
|
|
import os
|
|
import sys
|
|
from datetime import date
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
|
logger = logging.getLogger("finra_gap_fill")
|
|
|
|
_LOCK_FILE = "/tmp/backfill_finra_2020.lock"
|
|
|
|
# 2020-10 is partially ingested (15 days) → start from 2020-04-01, end 2020-11-01
|
|
_GAP_START = date(2020, 4, 1)
|
|
_GAP_END = date(2020, 11, 1)
|
|
|
|
|
|
async def run_gap_fill():
|
|
from app.core.database import AsyncSessionLocal
|
|
from app.services.finra_short_volume_service import FinraShortVolumeService
|
|
|
|
logger.info(f"Starting FINRA gap fill: {_GAP_START} → {_GAP_END}")
|
|
|
|
svc = FinraShortVolumeService()
|
|
async with AsyncSessionLocal() as db:
|
|
inserted_total = await svc.ingest_date_range(
|
|
db=db,
|
|
start_date=_GAP_START,
|
|
end_date=_GAP_END,
|
|
force_refresh=False, # skip already-ingested dates
|
|
)
|
|
|
|
logger.info("=" * 60)
|
|
logger.info("FINRA 2020 GAP FILL COMPLETE")
|
|
logger.info(f" Period : {_GAP_START} → {_GAP_END}")
|
|
logger.info(f" Inserted : {inserted_total:,} rows (0 = all dates already present or holiday)")
|
|
logger.info("=" * 60)
|
|
logger.info("Verify with:")
|
|
logger.info(" SELECT to_char(date,'YYYY-MM'), COUNT(DISTINCT date::date)")
|
|
logger.info(" FROM finra_short_volume")
|
|
logger.info(" WHERE date >= '2020-03-01' AND date < '2021-02-01'")
|
|
logger.info(" GROUP BY 1 ORDER BY 1;")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
lock_fh = open(_LOCK_FILE, "w")
|
|
try:
|
|
fcntl.flock(lock_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except OSError:
|
|
logger.error("Gap fill is already running (lock file held). Exiting.")
|
|
sys.exit(1)
|
|
|
|
try:
|
|
asyncio.run(run_gap_fill())
|
|
finally:
|
|
fcntl.flock(lock_fh, fcntl.LOCK_UN)
|
|
lock_fh.close()
|
|
try:
|
|
os.unlink(_LOCK_FILE)
|
|
except OSError:
|
|
pass
|