yirugi
/
stock-oracle
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
main
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from '27990ea908'
${ noResults }
stock-oracle/app/services/sec_filings_service.py

634 lines
26 KiB
Python
Raw Blame History

"""
SEC Filings Service: index, search, and retrieve SEC filings (8-K, 6-K, 20-F, 40-F, etc.)
and extract exhibit content (e.g., EX-99.1 press releases).
"""
import asyncio
import logging
import time
import uuid
from datetime import datetime, timezone
from typing import Dict, List, Optional, Set, Tuple
from app.core.config import settings
from bs4 import BeautifulSoup
from sqlalchemy import select, and_, func
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.filing import SECFiling
from app.services.sec_http_client import SECHttpClient
logger = logging.getLogger(__name__)
# Maximum exhibit content size (5 MB)
MAX_EXHIBIT_SIZE = 5 * 1024 * 1024
# Limit concurrent background reindex tasks to prevent event-loop saturation
# when bulk scans trigger hundreds of simultaneous SEC EDGAR fetches.
_BACKGROUND_REINDEX_SEMAPHORE = asyncio.Semaphore(3)
class SECFilingsService:
SUPPORTED_FORM_TYPES: Set[str] = {
"8-K", "6-K", "20-F", "40-F", "10-K", "10-Q",
"8-K/A", "6-K/A", "20-F/A", "40-F/A",
}
CHUNK_SIZE = 500
def __init__(self):
self._http = SECHttpClient("Stock Oracle Filings Service")
self._doc_fetch_locks: Dict[str, asyncio.Lock] = {}
self._reindex_locks: Dict[str, asyncio.Lock] = {}
# ------------------------------------------------------------------
# index_filings: fetch from SEC submissions and upsert to DB
# ------------------------------------------------------------------
async def index_filings(
self,
db: AsyncSession,
ticker: str,
form_types: Optional[Set[str]] = None,
force_refresh: bool = False,
skip_cache: bool = False,
) -> int:
"""Index filings from SEC submissions endpoint into sec_filings table.
Returns the number of filings indexed (inserted or updated).
Args:
force_refresh: Update existing DB records in addition to inserting new ones.
Also implies skip_cache=True.
skip_cache: Bypass HTTP cache reads to fetch fresh data from SEC EDGAR.
"""
ticker = ticker.upper()
# force_refresh always implies fresh data from SEC
effective_skip_cache = skip_cache or force_refresh
self._http.set_deadline(120.0)
try:
cik = await self._http.get_company_cik(ticker)
if not cik:
raise ValueError(f"Could not find CIK for ticker {ticker}")
cik_int = int(cik)
url = f"{self._http.sec_base_data}/submissions/CIK{cik_int:010d}.json"
data = await self._http.fetch_json(url, skip_cache=effective_skip_cache)
# Determine which form types to index
target_forms = form_types or self.SUPPORTED_FORM_TYPES
# Collect filings from recent block
raw_filings: List[Dict] = []
def add_from_block(block: dict) -> None:
forms = block.get("form", [])
dates = block.get("filingDate", [])
accessions = block.get("accessionNumber", [])
primary_docs = block.get("primaryDocument", [])
descriptions = block.get("primaryDocDescription", [])
accepted_dates = block.get("acceptanceDateTime", [])
for i, (form, dt_str, acc) in enumerate(zip(forms, dates, accessions)):
if form not in target_forms:
continue
pri_doc = primary_docs[i] if i < len(primary_docs) else None
desc = descriptions[i] if i < len(descriptions) else None
try:
filing_date = datetime.strptime(dt_str, "%Y-%m-%d").replace(
tzinfo=timezone.utc
)
except Exception:
continue
accepted_at = None
if i < len(accepted_dates) and accepted_dates[i]:
try:
accepted_at = datetime.fromisoformat(
accepted_dates[i].replace("Z", "+00:00")
)
except Exception:
pass
acc_clean = acc.replace("-", "")
pri_doc_url = None
if pri_doc:
pri_doc_url = (
f"https://www.sec.gov/Archives/edgar/data/{cik_int}/{acc_clean}/{pri_doc}"
)
raw_filings.append({
"ticker": ticker,
"cik": cik,
"accession_number": acc,
"form_type": form,
"filing_date": filing_date,
"accepted_at": accepted_at,
"primary_document": pri_doc,
"primary_document_url": pri_doc_url,
"filing_description": desc,
})
recent = data.get("filings", {}).get("recent", {})
add_from_block(recent)
# Fetch older yearly submission files
files_meta = data.get("filings", {}).get("files", []) or []
for meta in files_meta[:6]:
name = meta.get("name")
if not name:
continue
older_url = f"{self._http.sec_base_data}/submissions/{name}"
try:
older = await self._http.fetch_json(older_url, skip_cache=effective_skip_cache)
add_from_block(older)
except Exception:
continue
if not raw_filings:
return 0
# Snapshot pre-existing accessions to identify truly new inserts later
acc_result = await db.execute(
select(SECFiling.accession_number).where(SECFiling.ticker == ticker)
)
_pre_existing_accs: Set[str] = {row[0] for row in acc_result.fetchall()}
# Deduplicate within the same SEC response (accession_number may repeat)
seen: Set[str] = set()
deduped: List[Dict] = []
for rf in raw_filings:
if rf["accession_number"] not in seen:
seen.add(rf["accession_number"])
deduped.append(rf)
raw_filings = deduped
now = datetime.now(timezone.utc)
indexed_count = 0
# Upsert in chunks (ON CONFLICT eliminates race conditions)
for chunk_start in range(0, len(raw_filings), self.CHUNK_SIZE):
chunk = raw_filings[chunk_start: chunk_start + self.CHUNK_SIZE]
rows = [
{
"id": uuid.uuid4(),
"ticker": rf["ticker"],
"cik": rf["cik"],
"accession_number": rf["accession_number"],
"form_type": rf["form_type"],
"filing_date": rf["filing_date"],
"accepted_at": rf["accepted_at"],
"primary_document": rf["primary_document"],
"primary_document_url": rf["primary_document_url"],
"filing_description": rf["filing_description"],
"parsed_status": "pending" if rf["form_type"] in ("8-K", "8-K/A") else None,
"indexed_at": now,
"created_at": now,
"updated_at": now,
}
for rf in chunk
]
stmt = pg_insert(SECFiling).values(rows)
if force_refresh:
stmt = stmt.on_conflict_do_update(
index_elements=["accession_number"],
set_={
"ticker": stmt.excluded.ticker,
"cik": stmt.excluded.cik,
"form_type": stmt.excluded.form_type,
"filing_date": stmt.excluded.filing_date,
"accepted_at": stmt.excluded.accepted_at,
"primary_document": stmt.excluded.primary_document,
"primary_document_url": stmt.excluded.primary_document_url,
"filing_description": stmt.excluded.filing_description,
"indexed_at": stmt.excluded.indexed_at,
"updated_at": stmt.excluded.updated_at,
},
)
else:
stmt = stmt.on_conflict_do_nothing(
index_elements=["accession_number"],
)
try:
result = await db.execute(stmt)
await db.commit()
indexed_count += result.rowcount
except Exception as e:
logger.error(
f"Chunk upsert failed at offset {chunk_start} for {ticker}: {e}"
)
await db.rollback()
logger.info(f"Indexed {indexed_count} filings for {ticker}")
# Auto-parse newly inserted 8-K filings — cap at 20 most recent
# to avoid blocking for tickers with 1000+ historical 8-Ks.
# Remaining pending filings are lazy-parsed via /events endpoint.
truly_new_8k = [
rf["accession_number"]
for rf in raw_filings
if rf["form_type"] in ("8-K", "8-K/A")
and rf["accession_number"] not in _pre_existing_accs
]
if truly_new_8k:
# raw_filings is ordered newest-first from SEC, so slice keeps recent ones
parse_batch = truly_new_8k[:20]
from app.services.sec_8k_parser import sec_8k_parser
try:
await sec_8k_parser.parse_filings_by_accessions(db, parse_batch)
except Exception as e:
logger.warning(f"Auto-parse failed for {ticker} 8-Ks: {e}")
return indexed_count
finally:
self._http.clear_deadline()
# ------------------------------------------------------------------
# Background staleness re-index (non-blocking)
# ------------------------------------------------------------------
async def _background_reindex(
self, ticker: str, form_types: Optional[Set[str]], lock: asyncio.Lock
) -> None:
"""Re-index a stale ticker in the background with its own DB session."""
async with _BACKGROUND_REINDEX_SEMAPHORE:
async with lock:
try:
from app.core.database import AsyncSessionLocal
async with AsyncSessionLocal() as session:
new_count = await self.index_filings(
session, ticker, form_types=form_types, skip_cache=True
)
logger.info(f"Background reindex for {ticker}: {new_count} new filings")
except Exception as e:
logger.warning(f"Background reindex failed for {ticker}: {e}")
# ------------------------------------------------------------------
# search_filings: query DB with filters
# ------------------------------------------------------------------
async def search_filings(
self,
db: AsyncSession,
ticker: str,
form_types: Optional[Set[str]] = None,
start_date: Optional[datetime] = None,
end_date: Optional[datetime] = None,
limit: int = 20,
offset: int = 0,
) -> Tuple[List[SECFiling], int]:
"""Search indexed filings. Auto-indexes if none found."""
ticker = ticker.upper()
conditions = [SECFiling.ticker == ticker]
if form_types:
conditions.append(SECFiling.form_type.in_(form_types))
if start_date:
conditions.append(SECFiling.filing_date >= start_date)
if end_date:
conditions.append(SECFiling.filing_date <= end_date)
where_clause = and_(*conditions)
# Count
count_result = await db.execute(
select(func.count()).select_from(SECFiling).where(where_clause)
)
total_count = count_result.scalar() or 0
# If no results, try indexing first
if total_count == 0:
try:
await self.index_filings(db, ticker, form_types=form_types)
except Exception as e:
logger.warning(f"Auto-indexing failed for {ticker}: {e}")
return [], 0
# Re-count after indexing
count_result = await db.execute(
select(func.count()).select_from(SECFiling).where(where_clause)
)
total_count = count_result.scalar() or 0
if total_count == 0:
return [], 0
else:
# Ticker already has filings — check for staleness and fire background re-index.
# Returns stale data immediately; the background task updates for next requests.
staleness_result = await db.execute(
select(func.max(SECFiling.indexed_at)).where(SECFiling.ticker == ticker)
)
last_indexed = staleness_result.scalar()
now_utc = datetime.now(timezone.utc)
refresh_seconds = settings.SEC_DATA_REFRESH_HOURS * 3600
is_stale = last_indexed is None or (
(now_utc - last_indexed).total_seconds() > refresh_seconds
)
if is_stale:
if ticker not in self._reindex_locks:
self._reindex_locks[ticker] = asyncio.Lock()
lock = self._reindex_locks[ticker]
if not lock.locked():
# Touch indexed_at immediately to prevent re-trigger from
# concurrent requests while background task runs.
await db.execute(
SECFiling.__table__.update()
.where(
SECFiling.id.in_(
select(SECFiling.id)
.where(SECFiling.ticker == ticker)
.order_by(SECFiling.filing_date.desc())
.limit(1)
)
)
.values(indexed_at=now_utc)
)
await db.commit()
# Fire background re-index with its own DB session
asyncio.ensure_future(
self._background_reindex(ticker, form_types, lock)
)
# Query with pagination
result = await db.execute(
select(SECFiling)
.where(where_clause)
.order_by(SECFiling.filing_date.desc())
.limit(limit)
.offset(offset)
)
filings = result.scalars().all()
return list(filings), total_count
# ------------------------------------------------------------------
# get_filing_documents: list all documents in a filing
# ------------------------------------------------------------------
async def get_filing_documents(
self, db: AsyncSession, accession_number: str
) -> List[Dict]:
"""Get list of documents for a filing. Caches in documents_json column."""
_owned_deadline = self._http.remaining_time() is None
if _owned_deadline:
self._http.set_deadline(30.0)
try:
result = await db.execute(
select(SECFiling).where(SECFiling.accession_number == accession_number)
)
filing = result.scalar_one_or_none()
if not filing:
raise ValueError(f"Filing {accession_number} not found in database")
# Return cached documents if available
if filing.documents_json is not None:
return filing.documents_json
# Singleflight: serialize concurrent fetches for the same accession number
if accession_number not in self._doc_fetch_locks:
self._doc_fetch_locks[accession_number] = asyncio.Lock()
lock = self._doc_fetch_locks[accession_number]
async with lock:
# Re-check after acquiring lock (another coroutine may have fetched already)
result = await db.execute(
select(SECFiling).where(SECFiling.accession_number == accession_number)
)
filing = result.scalar_one_or_none()
if filing and filing.documents_json is not None:
return filing.documents_json
# Fetch and parse index page
cik_int = int("".join(ch for ch in filing.cik if ch.isdigit()))
acc_clean = accession_number.replace("-", "")
index_url = (
f"https://www.sec.gov/Archives/edgar/data/{cik_int}"
f"/{acc_clean}/{accession_number}-index.htm"
)
html = await self._http.fetch_text(index_url)
soup = BeautifulSoup(html, "html.parser")
base_url = index_url.rsplit("/", 1)[0]
def mk_abs(href: str) -> str:
if href.startswith("http"):
return href
if href.startswith("/"):
return f"https://www.sec.gov{href}"
return f"{base_url}/{href}"
documents: List[Dict] = []
for row in soup.find_all("tr"):
cells = row.find_all(["td", "th"])
if len(cells) < 4:
continue
# Typical columns: Seq, Description, Document, Type, Size
desc = cells[1].get_text(strip=True) if len(cells) > 1 else ""
doc_cell = cells[2]
doc_type = cells[3].get_text(strip=True) if len(cells) > 3 else ""
size_text = cells[4].get_text(strip=True) if len(cells) > 4 else ""
a_tag = doc_cell.find("a")
if not a_tag or not a_tag.get("href"):
continue
href = a_tag["href"]
filename = a_tag.get_text(strip=True) or doc_cell.get_text(strip=True)
documents.append({
"type": doc_type,
"description": desc,
"filename": filename,
"url": mk_abs(href),
"size": size_text,
})
# Cache to DB (always, even if empty, to prevent repeated SEC fetches)
filing.documents_json = documents
filing.updated_at = datetime.now(timezone.utc)
await db.commit()
return documents
finally:
if _owned_deadline:
self._http.clear_deadline()
# ------------------------------------------------------------------
# get_exhibit_content: extract exhibit text
# ------------------------------------------------------------------
async def get_exhibit_content(
self,
db: AsyncSession,
accession_number: str,
exhibit_type: str = "EX-99.1",
) -> Dict:
"""Download and return the content of a specific exhibit."""
self._http.set_deadline(25.0)
try:
documents = await self.get_filing_documents(db, accession_number)
if not documents:
raise ValueError(f"No documents found for filing {accession_number}")
exhibit_type_upper = exhibit_type.upper()
# Find matching document by type
target = None
for doc in documents:
doc_type = (doc.get("type") or "").upper()
if doc_type == exhibit_type_upper:
target = doc
break
# Fallback: try matching description
if not target:
for doc in documents:
desc = (doc.get("description") or "").upper()
if exhibit_type_upper in desc:
target = doc
break
if not target:
raise ValueError(
f"Exhibit {exhibit_type} not found in filing {accession_number}. "
f"Available types: {[d.get('type') for d in documents]}"
)
url = target["url"]
content = await self._http.fetch_text(url, max_bytes=MAX_EXHIBIT_SIZE)
# Determine content type
filename = target.get("filename", "")
if filename.lower().endswith(".htm") or filename.lower().endswith(".html"):
content_type = "text/html"
elif filename.lower().endswith(".xml"):
content_type = "application/xml"
else:
content_type = "text/plain"
return {
"content": content,
"content_type": content_type,
"filename": filename,
"url": url,
}
finally:
self._http.clear_deadline()
# ------------------------------------------------------------------
# search_filings_bulk: bulk filing search for multiple tickers
# ------------------------------------------------------------------
async def search_filings_bulk(
self,
db: AsyncSession,
tickers: List[str],
form_types: Optional[Set[str]] = None,
start_date: Optional[datetime] = None,
end_date: Optional[datetime] = None,
limit_per_ticker: int = 20,
) -> List[Dict]:
"""Bulk search filings for multiple tickers. Auto-indexes missing tickers in parallel."""
t0 = time.monotonic()
tickers = [t.upper() for t in tickers]
# Identify which tickers already have indexed filings
conditions = [SECFiling.ticker.in_(tickers)]
if form_types:
conditions.append(SECFiling.form_type.in_(form_types))
count_result = await db.execute(
select(SECFiling.ticker, func.count().label("cnt"))
.where(and_(*conditions))
.group_by(SECFiling.ticker)
)
indexed_counts = {row.ticker: row.cnt for row in count_result}
missing = [t for t in tickers if indexed_counts.get(t, 0) == 0]
# Check staleness for tickers that already have filings
stale: List[str] = []
present_tickers = [t for t in tickers if indexed_counts.get(t, 0) > 0]
if present_tickers:
staleness_rows = await db.execute(
select(SECFiling.ticker, func.max(SECFiling.indexed_at))
.where(SECFiling.ticker.in_(present_tickers))
.group_by(SECFiling.ticker)
)
now_utc = datetime.now(timezone.utc)
refresh_seconds = settings.SEC_DATA_REFRESH_HOURS * 3600
for row in staleness_rows:
t_name, last_indexed = row[0], row[1]
if last_indexed is None or (now_utc - last_indexed).total_seconds() > refresh_seconds:
stale.append(t_name)
# Index missing tickers and re-index stale tickers in parallel (Semaphore(4))
# Each coroutine uses its own session to avoid concurrent-session conflicts
to_index = [(t, False) for t in missing] + [(t, True) for t in stale]
if to_index:
sem = asyncio.Semaphore(4)
async def _index_one(ticker: str, is_stale: bool) -> None:
async with sem:
try:
from app.core.database import AsyncSessionLocal
async with AsyncSessionLocal() as session:
await asyncio.wait_for(
self.index_filings(
session, ticker, form_types=form_types,
skip_cache=is_stale,
),
timeout=120,
)
except Exception as e:
logger.warning(f"Bulk index failed for {ticker}: {e}")
await asyncio.gather(*[_index_one(t, s) for t, s in to_index])
# Fetch results for all tickers from DB
results = []
for ticker in tickers:
ticker_conditions = [SECFiling.ticker == ticker]
if form_types:
ticker_conditions.append(SECFiling.form_type.in_(form_types))
if start_date:
ticker_conditions.append(SECFiling.filing_date >= start_date)
if end_date:
ticker_conditions.append(SECFiling.filing_date <= end_date)
try:
count_res = await db.execute(
select(func.count())
.select_from(SECFiling)
.where(and_(*ticker_conditions))
)
total = count_res.scalar() or 0
rows = await db.execute(
select(SECFiling)
.where(and_(*ticker_conditions))
.order_by(SECFiling.filing_date.desc())
.limit(limit_per_ticker)
)
filings = rows.scalars().all()
results.append({
"ticker": ticker,
"success": True,
"filings": list(filings),
"total_count": total,
"error": None,
})
except Exception as e:
results.append({
"ticker": ticker,
"success": False,
"filings": [],
"total_count": 0,
"error": str(e),
})
elapsed = time.monotonic() - t0
return results, elapsed
# Module-level singleton
sec_filings_service = SECFilingsService()
Reference in New Issue View Git Blame Copy Permalink