Add SEC filings indexing, search, and exhibit extraction (8-K, 6-K, 20-F, 40-F)
Extract shared SECHttpClient from ETF fetcher (retry, backoff, cache, throttle) and apply it to both ETF and core SEC services, fixing missing rate limiting. Add SECFiling DB model, Pydantic schemas, SECFilingsService with auto-indexing, and REST endpoints at /filings/search, /filings/documents, /filings/exhibit. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>main
I Luk Kim
5 months ago
parent
44f8979805
commit
d0f1a9a7d0
@ -0,0 +1,162 @@
|
||||
"""
|
||||
SEC Filings endpoints: search, document listing, and exhibit extraction.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.database import get_db
|
||||
from app.schemas.filing import (
|
||||
ExhibitContentResponse,
|
||||
FilingDocumentInfo,
|
||||
FilingDocumentListResponse,
|
||||
FilingSearchResponse,
|
||||
FilingSummary,
|
||||
)
|
||||
from app.services.sec_filings_service import sec_filings_service
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger("app.api.v1.filings")
|
||||
|
||||
|
||||
@router.get("/search/{ticker}", response_model=FilingSearchResponse)
|
||||
async def search_filings(
|
||||
ticker: str,
|
||||
form_type: Optional[str] = Query(
|
||||
None,
|
||||
description="Comma-separated form types (e.g. '8-K,6-K'). Default: all supported.",
|
||||
),
|
||||
start_date: Optional[str] = Query(None, description="Start date YYYY-MM-DD"),
|
||||
end_date: Optional[str] = Query(None, description="End date YYYY-MM-DD"),
|
||||
limit: int = Query(20, ge=1, le=100),
|
||||
offset: int = Query(0, ge=0),
|
||||
force_refresh: bool = Query(False, description="Force re-indexing from SEC"),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Search SEC filings for a ticker. Auto-indexes from SEC if not in DB."""
|
||||
form_types_set = None
|
||||
if form_type:
|
||||
form_types_set = {ft.strip().upper() for ft in form_type.split(",") if ft.strip()}
|
||||
unsupported = form_types_set - sec_filings_service.SUPPORTED_FORM_TYPES
|
||||
if unsupported:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Unsupported form types: {unsupported}. Supported: {sec_filings_service.SUPPORTED_FORM_TYPES}",
|
||||
)
|
||||
|
||||
start_dt = None
|
||||
end_dt = None
|
||||
try:
|
||||
if start_date:
|
||||
start_dt = datetime.strptime(start_date, "%Y-%m-%d").replace(tzinfo=timezone.utc)
|
||||
if end_date:
|
||||
end_dt = datetime.strptime(end_date, "%Y-%m-%d").replace(
|
||||
hour=23, minute=59, second=59, tzinfo=timezone.utc
|
||||
)
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=400, detail="Invalid date format. Use YYYY-MM-DD")
|
||||
|
||||
if force_refresh:
|
||||
try:
|
||||
await sec_filings_service.index_filings(
|
||||
db, ticker, form_types=form_types_set, force_refresh=True
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=404, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.error(f"Force refresh indexing failed for {ticker}: {e}")
|
||||
raise HTTPException(status_code=502, detail=f"SEC indexing failed: {e}")
|
||||
|
||||
try:
|
||||
filings, total_count = await sec_filings_service.search_filings(
|
||||
db,
|
||||
ticker,
|
||||
form_types=form_types_set,
|
||||
start_date=start_dt,
|
||||
end_date=end_dt,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=404, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.error(f"Filing search failed for {ticker}: {e}")
|
||||
raise HTTPException(status_code=502, detail=f"Filing search failed: {e}")
|
||||
|
||||
summaries = []
|
||||
for f in filings:
|
||||
docs_count = len(f.documents_json) if f.documents_json else None
|
||||
summaries.append(
|
||||
FilingSummary(
|
||||
accession_number=f.accession_number,
|
||||
form_type=f.form_type,
|
||||
filing_date=f.filing_date.strftime("%Y-%m-%d"),
|
||||
primary_document=f.primary_document,
|
||||
filing_description=f.filing_description,
|
||||
documents_count=docs_count,
|
||||
)
|
||||
)
|
||||
|
||||
return FilingSearchResponse(
|
||||
ticker=ticker.upper(),
|
||||
filings=summaries,
|
||||
total_count=total_count,
|
||||
metadata={
|
||||
"limit": limit,
|
||||
"offset": offset,
|
||||
"form_types": list(form_types_set) if form_types_set else None,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/documents/{accession_number}", response_model=FilingDocumentListResponse)
|
||||
async def get_filing_documents(
|
||||
accession_number: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""List all documents within a SEC filing."""
|
||||
try:
|
||||
documents = await sec_filings_service.get_filing_documents(db, accession_number)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=404, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.error(f"Document listing failed for {accession_number}: {e}")
|
||||
raise HTTPException(status_code=502, detail=f"Document listing failed: {e}")
|
||||
|
||||
doc_infos = [FilingDocumentInfo(**d) for d in documents]
|
||||
return FilingDocumentListResponse(
|
||||
accession_number=accession_number,
|
||||
documents=doc_infos,
|
||||
metadata={"total_documents": len(doc_infos)},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/exhibit/{accession_number}", response_model=ExhibitContentResponse)
|
||||
async def get_exhibit_content(
|
||||
accession_number: str,
|
||||
exhibit_type: str = Query("EX-99.1", description="Exhibit type (e.g. EX-99.1)"),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Extract exhibit content (e.g., press release EX-99.1) from a filing."""
|
||||
try:
|
||||
result = await sec_filings_service.get_exhibit_content(
|
||||
db, accession_number, exhibit_type
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=404, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.error(f"Exhibit extraction failed for {accession_number}/{exhibit_type}: {e}")
|
||||
raise HTTPException(status_code=502, detail=f"Exhibit extraction failed: {e}")
|
||||
|
||||
return ExhibitContentResponse(
|
||||
accession_number=accession_number,
|
||||
exhibit_type=exhibit_type,
|
||||
content=result["content"],
|
||||
content_type=result.get("content_type"),
|
||||
filename=result.get("filename"),
|
||||
url=result.get("url"),
|
||||
)
|
||||
@ -0,0 +1,33 @@
|
||||
"""
|
||||
Database model for SEC filings (8-K, 6-K, 20-F, 40-F, 10-K, 10-Q, etc.)
|
||||
"""
|
||||
|
||||
from sqlalchemy import Column, String, JSON, Index
|
||||
from sqlalchemy.dialects.postgresql import UUID, TIMESTAMP
|
||||
from datetime import datetime, timezone
|
||||
import uuid
|
||||
from app.core.database import Base
|
||||
|
||||
|
||||
class SECFiling(Base):
|
||||
__tablename__ = "sec_filings"
|
||||
|
||||
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
ticker = Column(String(10), nullable=False, index=True)
|
||||
cik = Column(String(20), nullable=False, index=True)
|
||||
accession_number = Column(String(30), nullable=False, unique=True, index=True)
|
||||
form_type = Column(String(20), nullable=False, index=True)
|
||||
filing_date = Column(TIMESTAMP(timezone=True), nullable=False)
|
||||
primary_document = Column(String(256))
|
||||
primary_document_url = Column(String(512))
|
||||
filing_description = Column(String(512))
|
||||
documents_json = Column(JSON) # Cached document list
|
||||
indexed_at = Column(TIMESTAMP(timezone=True))
|
||||
created_at = Column(TIMESTAMP(timezone=True), default=lambda: datetime.now(timezone.utc))
|
||||
updated_at = Column(TIMESTAMP(timezone=True), default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc))
|
||||
|
||||
__table_args__ = (
|
||||
Index("idx_sec_filings_ticker_form", "ticker", "form_type"),
|
||||
Index("idx_sec_filings_ticker_date", "ticker", "filing_date"),
|
||||
Index("idx_sec_filings_cik_form", "cik", "form_type"),
|
||||
)
|
||||
@ -0,0 +1,46 @@
|
||||
"""
|
||||
Pydantic schemas for SEC filing endpoints.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import List, Optional
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class FilingDocumentInfo(BaseModel):
|
||||
type: Optional[str] = None
|
||||
description: Optional[str] = None
|
||||
filename: Optional[str] = None
|
||||
url: Optional[str] = None
|
||||
size: Optional[str] = None
|
||||
|
||||
|
||||
class FilingSummary(BaseModel):
|
||||
accession_number: str
|
||||
form_type: str
|
||||
filing_date: str
|
||||
primary_document: Optional[str] = None
|
||||
filing_description: Optional[str] = None
|
||||
documents_count: Optional[int] = None
|
||||
|
||||
|
||||
class FilingSearchResponse(BaseModel):
|
||||
ticker: str
|
||||
filings: List[FilingSummary]
|
||||
total_count: int
|
||||
metadata: dict = Field(default_factory=dict)
|
||||
|
||||
|
||||
class FilingDocumentListResponse(BaseModel):
|
||||
accession_number: str
|
||||
documents: List[FilingDocumentInfo]
|
||||
metadata: dict = Field(default_factory=dict)
|
||||
|
||||
|
||||
class ExhibitContentResponse(BaseModel):
|
||||
accession_number: str
|
||||
exhibit_type: str
|
||||
content: str
|
||||
content_type: Optional[str] = None
|
||||
filename: Optional[str] = None
|
||||
url: Optional[str] = None
|
||||
@ -0,0 +1,354 @@
|
||||
"""
|
||||
SEC Filings Service: index, search, and retrieve SEC filings (8-K, 6-K, 20-F, 40-F, etc.)
|
||||
and extract exhibit content (e.g., EX-99.1 press releases).
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Dict, List, Optional, Set, Tuple
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
from sqlalchemy import select, and_, func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.filing import SECFiling
|
||||
from app.services.sec_http_client import SECHttpClient
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Maximum exhibit content size (1 MB)
|
||||
MAX_EXHIBIT_SIZE = 1 * 1024 * 1024
|
||||
|
||||
|
||||
class SECFilingsService:
|
||||
SUPPORTED_FORM_TYPES: Set[str] = {
|
||||
"8-K", "6-K", "20-F", "40-F", "10-K", "10-Q",
|
||||
"8-K/A", "6-K/A", "20-F/A", "40-F/A",
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
self._http = SECHttpClient("Stock Oracle Filings Service")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# index_filings: fetch from SEC submissions and upsert to DB
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def index_filings(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
ticker: str,
|
||||
form_types: Optional[Set[str]] = None,
|
||||
force_refresh: bool = False,
|
||||
) -> int:
|
||||
"""Index filings from SEC submissions endpoint into sec_filings table.
|
||||
|
||||
Returns the number of filings indexed (inserted or updated).
|
||||
"""
|
||||
ticker = ticker.upper()
|
||||
cik = await self._http.get_company_cik(ticker)
|
||||
if not cik:
|
||||
raise ValueError(f"Could not find CIK for ticker {ticker}")
|
||||
|
||||
cik_int = int(cik)
|
||||
url = f"{self._http.sec_base_data}/submissions/CIK{cik_int:010d}.json"
|
||||
data = await self._http.fetch_json(url)
|
||||
|
||||
# Determine which form types to index
|
||||
target_forms = form_types or self.SUPPORTED_FORM_TYPES
|
||||
|
||||
# Collect filings from recent block
|
||||
raw_filings: List[Dict] = []
|
||||
|
||||
def add_from_block(block: dict) -> None:
|
||||
forms = block.get("form", [])
|
||||
dates = block.get("filingDate", [])
|
||||
accessions = block.get("accessionNumber", [])
|
||||
primary_docs = block.get("primaryDocument", [])
|
||||
descriptions = block.get("primaryDocDescription", [])
|
||||
for i, (form, dt_str, acc) in enumerate(zip(forms, dates, accessions)):
|
||||
if form not in target_forms:
|
||||
continue
|
||||
pri_doc = primary_docs[i] if i < len(primary_docs) else None
|
||||
desc = descriptions[i] if i < len(descriptions) else None
|
||||
try:
|
||||
filing_date = datetime.strptime(dt_str, "%Y-%m-%d").replace(
|
||||
tzinfo=timezone.utc
|
||||
)
|
||||
except Exception:
|
||||
continue
|
||||
acc_clean = acc.replace("-", "")
|
||||
pri_doc_url = None
|
||||
if pri_doc:
|
||||
pri_doc_url = (
|
||||
f"https://www.sec.gov/Archives/edgar/data/{cik_int}/{acc_clean}/{pri_doc}"
|
||||
)
|
||||
raw_filings.append({
|
||||
"ticker": ticker,
|
||||
"cik": cik,
|
||||
"accession_number": acc,
|
||||
"form_type": form,
|
||||
"filing_date": filing_date,
|
||||
"primary_document": pri_doc,
|
||||
"primary_document_url": pri_doc_url,
|
||||
"filing_description": desc,
|
||||
})
|
||||
|
||||
recent = data.get("filings", {}).get("recent", {})
|
||||
add_from_block(recent)
|
||||
|
||||
# Fetch older yearly submission files
|
||||
files_meta = data.get("filings", {}).get("files", []) or []
|
||||
for meta in files_meta[:6]:
|
||||
name = meta.get("name")
|
||||
if not name:
|
||||
continue
|
||||
older_url = f"{self._http.sec_base_data}/submissions/{name}"
|
||||
try:
|
||||
older = await self._http.fetch_json(older_url)
|
||||
add_from_block(older)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if not raw_filings:
|
||||
return 0
|
||||
|
||||
# Upsert into DB
|
||||
indexed_count = 0
|
||||
now = datetime.now(timezone.utc)
|
||||
for rf in raw_filings:
|
||||
acc = rf["accession_number"]
|
||||
result = await db.execute(
|
||||
select(SECFiling).where(SECFiling.accession_number == acc)
|
||||
)
|
||||
existing = result.scalar_one_or_none()
|
||||
if existing:
|
||||
if force_refresh:
|
||||
existing.ticker = rf["ticker"]
|
||||
existing.cik = rf["cik"]
|
||||
existing.form_type = rf["form_type"]
|
||||
existing.filing_date = rf["filing_date"]
|
||||
existing.primary_document = rf["primary_document"]
|
||||
existing.primary_document_url = rf["primary_document_url"]
|
||||
existing.filing_description = rf["filing_description"]
|
||||
existing.indexed_at = now
|
||||
existing.updated_at = now
|
||||
indexed_count += 1
|
||||
else:
|
||||
filing = SECFiling(
|
||||
ticker=rf["ticker"],
|
||||
cik=rf["cik"],
|
||||
accession_number=rf["accession_number"],
|
||||
form_type=rf["form_type"],
|
||||
filing_date=rf["filing_date"],
|
||||
primary_document=rf["primary_document"],
|
||||
primary_document_url=rf["primary_document_url"],
|
||||
filing_description=rf["filing_description"],
|
||||
indexed_at=now,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
db.add(filing)
|
||||
indexed_count += 1
|
||||
|
||||
if indexed_count:
|
||||
await db.commit()
|
||||
logger.info(f"Indexed {indexed_count} filings for {ticker}")
|
||||
return indexed_count
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# search_filings: query DB with filters
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def search_filings(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
ticker: str,
|
||||
form_types: Optional[Set[str]] = None,
|
||||
start_date: Optional[datetime] = None,
|
||||
end_date: Optional[datetime] = None,
|
||||
limit: int = 20,
|
||||
offset: int = 0,
|
||||
) -> Tuple[List[SECFiling], int]:
|
||||
"""Search indexed filings. Auto-indexes if none found."""
|
||||
ticker = ticker.upper()
|
||||
|
||||
conditions = [SECFiling.ticker == ticker]
|
||||
if form_types:
|
||||
conditions.append(SECFiling.form_type.in_(form_types))
|
||||
if start_date:
|
||||
conditions.append(SECFiling.filing_date >= start_date)
|
||||
if end_date:
|
||||
conditions.append(SECFiling.filing_date <= end_date)
|
||||
|
||||
where_clause = and_(*conditions)
|
||||
|
||||
# Count
|
||||
count_result = await db.execute(
|
||||
select(func.count()).select_from(SECFiling).where(where_clause)
|
||||
)
|
||||
total_count = count_result.scalar() or 0
|
||||
|
||||
# If no results, try indexing first
|
||||
if total_count == 0:
|
||||
try:
|
||||
await self.index_filings(db, ticker, form_types=form_types)
|
||||
except Exception as e:
|
||||
logger.warning(f"Auto-indexing failed for {ticker}: {e}")
|
||||
return [], 0
|
||||
|
||||
# Re-count after indexing
|
||||
count_result = await db.execute(
|
||||
select(func.count()).select_from(SECFiling).where(where_clause)
|
||||
)
|
||||
total_count = count_result.scalar() or 0
|
||||
if total_count == 0:
|
||||
return [], 0
|
||||
|
||||
# Query with pagination
|
||||
result = await db.execute(
|
||||
select(SECFiling)
|
||||
.where(where_clause)
|
||||
.order_by(SECFiling.filing_date.desc())
|
||||
.limit(limit)
|
||||
.offset(offset)
|
||||
)
|
||||
filings = result.scalars().all()
|
||||
return list(filings), total_count
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# get_filing_documents: list all documents in a filing
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def get_filing_documents(
|
||||
self, db: AsyncSession, accession_number: str
|
||||
) -> List[Dict]:
|
||||
"""Get list of documents for a filing. Caches in documents_json column."""
|
||||
result = await db.execute(
|
||||
select(SECFiling).where(SECFiling.accession_number == accession_number)
|
||||
)
|
||||
filing = result.scalar_one_or_none()
|
||||
if not filing:
|
||||
raise ValueError(f"Filing {accession_number} not found in database")
|
||||
|
||||
# Return cached documents if available
|
||||
if filing.documents_json:
|
||||
return filing.documents_json
|
||||
|
||||
# Fetch and parse index page
|
||||
cik_int = int("".join(ch for ch in filing.cik if ch.isdigit()))
|
||||
acc_clean = accession_number.replace("-", "")
|
||||
index_url = (
|
||||
f"https://www.sec.gov/Archives/edgar/data/{cik_int}"
|
||||
f"/{acc_clean}/{accession_number}-index.htm"
|
||||
)
|
||||
|
||||
html = await self._http.fetch_text(index_url)
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
|
||||
base_url = index_url.rsplit("/", 1)[0]
|
||||
|
||||
def mk_abs(href: str) -> str:
|
||||
if href.startswith("http"):
|
||||
return href
|
||||
if href.startswith("/"):
|
||||
return f"https://www.sec.gov{href}"
|
||||
return f"{base_url}/{href}"
|
||||
|
||||
documents: List[Dict] = []
|
||||
for row in soup.find_all("tr"):
|
||||
cells = row.find_all(["td", "th"])
|
||||
if len(cells) < 4:
|
||||
continue
|
||||
# Typical columns: Seq, Description, Document, Type, Size
|
||||
desc = cells[1].get_text(strip=True) if len(cells) > 1 else ""
|
||||
doc_cell = cells[2]
|
||||
doc_type = cells[3].get_text(strip=True) if len(cells) > 3 else ""
|
||||
size_text = cells[4].get_text(strip=True) if len(cells) > 4 else ""
|
||||
|
||||
a_tag = doc_cell.find("a")
|
||||
if not a_tag or not a_tag.get("href"):
|
||||
continue
|
||||
href = a_tag["href"]
|
||||
filename = a_tag.get_text(strip=True) or doc_cell.get_text(strip=True)
|
||||
|
||||
documents.append({
|
||||
"type": doc_type,
|
||||
"description": desc,
|
||||
"filename": filename,
|
||||
"url": mk_abs(href),
|
||||
"size": size_text,
|
||||
})
|
||||
|
||||
# Cache to DB
|
||||
if documents:
|
||||
filing.documents_json = documents
|
||||
filing.updated_at = datetime.now(timezone.utc)
|
||||
await db.commit()
|
||||
|
||||
return documents
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# get_exhibit_content: extract exhibit text
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def get_exhibit_content(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
accession_number: str,
|
||||
exhibit_type: str = "EX-99.1",
|
||||
) -> Dict:
|
||||
"""Download and return the content of a specific exhibit."""
|
||||
documents = await self.get_filing_documents(db, accession_number)
|
||||
if not documents:
|
||||
raise ValueError(f"No documents found for filing {accession_number}")
|
||||
|
||||
exhibit_type_upper = exhibit_type.upper()
|
||||
# Find matching document by type
|
||||
target = None
|
||||
for doc in documents:
|
||||
doc_type = (doc.get("type") or "").upper()
|
||||
if doc_type == exhibit_type_upper:
|
||||
target = doc
|
||||
break
|
||||
|
||||
# Fallback: try matching description
|
||||
if not target:
|
||||
for doc in documents:
|
||||
desc = (doc.get("description") or "").upper()
|
||||
if exhibit_type_upper in desc:
|
||||
target = doc
|
||||
break
|
||||
|
||||
if not target:
|
||||
raise ValueError(
|
||||
f"Exhibit {exhibit_type} not found in filing {accession_number}. "
|
||||
f"Available types: {[d.get('type') for d in documents]}"
|
||||
)
|
||||
|
||||
url = target["url"]
|
||||
content = await self._http.fetch_text(url)
|
||||
|
||||
if len(content) > MAX_EXHIBIT_SIZE:
|
||||
raise ValueError(
|
||||
f"Exhibit content exceeds size limit ({len(content)} bytes > {MAX_EXHIBIT_SIZE} bytes)"
|
||||
)
|
||||
|
||||
# Determine content type
|
||||
filename = target.get("filename", "")
|
||||
if filename.lower().endswith(".htm") or filename.lower().endswith(".html"):
|
||||
content_type = "text/html"
|
||||
elif filename.lower().endswith(".xml"):
|
||||
content_type = "application/xml"
|
||||
else:
|
||||
content_type = "text/plain"
|
||||
|
||||
return {
|
||||
"content": content,
|
||||
"content_type": content_type,
|
||||
"filename": filename,
|
||||
"url": url,
|
||||
}
|
||||
|
||||
|
||||
# Module-level singleton
|
||||
sec_filings_service = SECFilingsService()
|
||||
@ -0,0 +1,327 @@
|
||||
"""
|
||||
Shared SEC EDGAR HTTP client with retry, backoff, throttling, and caching.
|
||||
|
||||
Extracted from etf_holdings_fetcher.py to be reused by all SEC-related services.
|
||||
"""
|
||||
|
||||
import time as _time
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import logging
|
||||
from typing import Dict, Optional
|
||||
|
||||
from app.core.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _is_sec_block_page(text: str) -> bool:
|
||||
if not text:
|
||||
return False
|
||||
tl = text.lower()
|
||||
if "your request originates from an undeclared automated tool" in tl:
|
||||
return True
|
||||
if "<title>sec.gov | your request originates" in tl:
|
||||
return True
|
||||
if "reference id:" in tl and "sec.gov" in tl:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class SECHttpClient:
|
||||
"""Shared HTTP client for SEC EDGAR API requests.
|
||||
|
||||
Features:
|
||||
- asyncio.Semaphore(2) concurrent request limit
|
||||
- 6 retries with 1.8x exponential backoff + random jitter
|
||||
- 429 Retry-After header respect
|
||||
- SEC block page detection
|
||||
- Deadline-aware timeouts
|
||||
- Disk cache (/tmp/stock_oracle_sec_cache/, SHA-256 keyed)
|
||||
- In-memory cache (_json_cache, _text_cache)
|
||||
"""
|
||||
|
||||
def __init__(self, user_agent_name: str = "Stock Oracle"):
|
||||
self.sec_base_data = "https://data.sec.gov"
|
||||
self.sec_base_www = "https://www.sec.gov"
|
||||
self.http_timeout = aiohttp.ClientTimeout(total=12)
|
||||
self._req_sem = asyncio.Semaphore(2)
|
||||
self._text_cache: Dict[str, str] = {}
|
||||
self._json_cache: Dict[str, dict] = {}
|
||||
self._cache_dir = "/tmp/stock_oracle_sec_cache"
|
||||
try:
|
||||
os.makedirs(self._cache_dir, exist_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
self._user_agent = f"{user_agent_name} ({settings.SEC_EMAIL})"
|
||||
self._deadline: Optional[float] = None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Deadline management
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def set_deadline(self, seconds_from_now: float) -> None:
|
||||
self._deadline = _time.monotonic() + seconds_from_now
|
||||
|
||||
def clear_deadline(self) -> None:
|
||||
self._deadline = None
|
||||
|
||||
@property
|
||||
def deadline(self) -> Optional[float]:
|
||||
return self._deadline
|
||||
|
||||
@deadline.setter
|
||||
def deadline(self, value: Optional[float]) -> None:
|
||||
self._deadline = value
|
||||
|
||||
def remaining_time(self) -> Optional[float]:
|
||||
if self._deadline is None:
|
||||
return None
|
||||
return max(0.0, self._deadline - _time.monotonic())
|
||||
|
||||
def is_deadline_exceeded(self) -> bool:
|
||||
if self._deadline is None:
|
||||
return False
|
||||
return _time.monotonic() >= self._deadline
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Cache helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _cache_path(self, url: str) -> str:
|
||||
h = hashlib.sha256(url.encode("utf-8")).hexdigest()
|
||||
return os.path.join(self._cache_dir, h)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# CIK lookup
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def get_company_cik(self, ticker: str) -> Optional[str]:
|
||||
"""Look up zero-padded 10-digit CIK for a ticker."""
|
||||
url = f"{self.sec_base_www}/files/company_tickers.json"
|
||||
try:
|
||||
data = await self.fetch_json(url)
|
||||
for _key, company_info in data.items():
|
||||
if company_info.get("ticker", "").upper() == ticker.upper():
|
||||
cik_str = str(company_info.get("cik_str", "")).zfill(10)
|
||||
logger.info(f"Found CIK {cik_str} for ticker {ticker}")
|
||||
return cik_str
|
||||
logger.warning(f"Ticker {ticker} not found in SEC mapping")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching CIK for {ticker}: {e}")
|
||||
return None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# HTTP fetch methods
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def fetch_json(self, url: str) -> dict:
|
||||
"""Fetch JSON with retry, backoff, and caching."""
|
||||
# In-memory cache
|
||||
if url in self._json_cache:
|
||||
return self._json_cache[url]
|
||||
# Disk cache
|
||||
try:
|
||||
cp = self._cache_path(url) + ".json"
|
||||
if os.path.exists(cp):
|
||||
ttl_sec = max(3600, settings.SEC_DATA_REFRESH_HOURS * 3600)
|
||||
if _time.time() - os.path.getmtime(cp) <= ttl_sec:
|
||||
with open(cp, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
self._json_cache[url] = data
|
||||
return data
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
attempts = 6
|
||||
backoff = 1.0
|
||||
last_exc = None
|
||||
for _i in range(attempts):
|
||||
now = _time.monotonic()
|
||||
if self._deadline is not None and now >= self._deadline:
|
||||
break
|
||||
req_timeout = self.http_timeout
|
||||
if self._deadline is not None:
|
||||
remaining = max(0.0, self._deadline - now)
|
||||
if remaining < 0.25:
|
||||
break
|
||||
req_timeout = aiohttp.ClientTimeout(
|
||||
total=min(remaining, getattr(self.http_timeout, "total", 12))
|
||||
)
|
||||
async with self._req_sem:
|
||||
try:
|
||||
async with aiohttp.ClientSession(
|
||||
timeout=req_timeout,
|
||||
headers={
|
||||
"User-Agent": self._user_agent,
|
||||
"Accept": "application/json",
|
||||
},
|
||||
) as session:
|
||||
async with session.get(url, timeout=req_timeout) as resp:
|
||||
if resp.status == 429:
|
||||
retry_after = resp.headers.get("Retry-After")
|
||||
delay = (
|
||||
float(retry_after)
|
||||
if retry_after and retry_after.isdigit()
|
||||
else backoff
|
||||
)
|
||||
if self._deadline is not None:
|
||||
remaining = max(0.0, self._deadline - _time.monotonic())
|
||||
delay = min(delay, max(0.0, remaining - 0.05))
|
||||
await asyncio.sleep(
|
||||
max(0.0, delay)
|
||||
+ random.uniform(0.0, delay * 0.25 if delay > 0 else 0.0)
|
||||
)
|
||||
backoff *= 1.8
|
||||
continue
|
||||
if 500 <= resp.status < 600:
|
||||
delay = backoff
|
||||
if self._deadline is not None:
|
||||
remaining = max(0.0, self._deadline - _time.monotonic())
|
||||
delay = min(delay, max(0.0, remaining - 0.05))
|
||||
await asyncio.sleep(
|
||||
max(0.0, delay)
|
||||
+ random.uniform(0.0, delay * 0.25 if delay > 0 else 0.0)
|
||||
)
|
||||
backoff *= 1.8
|
||||
continue
|
||||
resp.raise_for_status()
|
||||
data = await resp.json()
|
||||
# Cache successful response
|
||||
try:
|
||||
with open(self._cache_path(url) + ".json", "w", encoding="utf-8") as f:
|
||||
json.dump(data, f)
|
||||
except Exception:
|
||||
pass
|
||||
self._json_cache[url] = data
|
||||
return data
|
||||
except Exception as e:
|
||||
last_exc = e
|
||||
delay = backoff
|
||||
if self._deadline is not None:
|
||||
remaining = max(0.0, self._deadline - _time.monotonic())
|
||||
delay = min(delay, max(0.0, remaining - 0.05))
|
||||
await asyncio.sleep(
|
||||
max(0.0, delay)
|
||||
+ random.uniform(0.0, delay * 0.25 if delay > 0 else 0.0)
|
||||
)
|
||||
backoff *= 1.8
|
||||
continue
|
||||
raise last_exc if last_exc else RuntimeError("Failed to fetch JSON")
|
||||
|
||||
async def fetch_text(self, url: str, accept: str = "text/html") -> str:
|
||||
"""Fetch text content with retry, backoff, block page detection, and caching."""
|
||||
# In-memory cache
|
||||
if url in self._text_cache:
|
||||
return self._text_cache[url]
|
||||
# Disk cache
|
||||
try:
|
||||
cp = self._cache_path(url) + ".txt"
|
||||
if os.path.exists(cp):
|
||||
ttl_sec = max(3600, settings.SEC_DATA_REFRESH_HOURS * 3600)
|
||||
if _time.time() - os.path.getmtime(cp) <= ttl_sec:
|
||||
with open(cp, "r", encoding="utf-8") as f:
|
||||
text = f.read()
|
||||
if _is_sec_block_page(text):
|
||||
try:
|
||||
os.remove(cp)
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
self._text_cache[url] = text
|
||||
return text
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
attempts = 6
|
||||
backoff = 1.0
|
||||
last_exc = None
|
||||
for _i in range(attempts):
|
||||
now = _time.monotonic()
|
||||
if self._deadline is not None and now >= self._deadline:
|
||||
break
|
||||
req_timeout = self.http_timeout
|
||||
if self._deadline is not None:
|
||||
remaining = max(0.0, self._deadline - now)
|
||||
if remaining < 0.25:
|
||||
break
|
||||
req_timeout = aiohttp.ClientTimeout(
|
||||
total=min(remaining, getattr(self.http_timeout, "total", 12))
|
||||
)
|
||||
async with self._req_sem:
|
||||
try:
|
||||
async with aiohttp.ClientSession(
|
||||
timeout=req_timeout,
|
||||
headers={
|
||||
"User-Agent": self._user_agent,
|
||||
"Accept": accept,
|
||||
},
|
||||
) as session:
|
||||
async with session.get(url, timeout=req_timeout) as resp:
|
||||
if resp.status == 429:
|
||||
retry_after = resp.headers.get("Retry-After")
|
||||
delay = (
|
||||
float(retry_after)
|
||||
if retry_after and retry_after.isdigit()
|
||||
else backoff
|
||||
)
|
||||
if self._deadline is not None:
|
||||
remaining = max(0.0, self._deadline - _time.monotonic())
|
||||
delay = min(delay, max(0.0, remaining - 0.05))
|
||||
await asyncio.sleep(
|
||||
max(0.0, delay)
|
||||
+ random.uniform(0.0, delay * 0.25 if delay > 0 else 0.0)
|
||||
)
|
||||
backoff *= 1.8
|
||||
continue
|
||||
if 500 <= resp.status < 600:
|
||||
delay = backoff
|
||||
if self._deadline is not None:
|
||||
remaining = max(0.0, self._deadline - _time.monotonic())
|
||||
delay = min(delay, max(0.0, remaining - 0.05))
|
||||
await asyncio.sleep(
|
||||
max(0.0, delay)
|
||||
+ random.uniform(0.0, delay * 0.25 if delay > 0 else 0.0)
|
||||
)
|
||||
backoff *= 1.8
|
||||
continue
|
||||
resp.raise_for_status()
|
||||
text = await resp.text()
|
||||
if _is_sec_block_page(text):
|
||||
last_exc = RuntimeError("SEC_BLOCKED")
|
||||
delay = backoff * 2.0
|
||||
if self._deadline is not None:
|
||||
remaining = max(0.0, self._deadline - _time.monotonic())
|
||||
delay = min(delay, max(0.0, remaining - 0.05))
|
||||
await asyncio.sleep(
|
||||
max(0.0, delay)
|
||||
+ random.uniform(0.0, delay * 0.5 if delay > 0 else 0.0)
|
||||
)
|
||||
backoff *= 2.0
|
||||
continue
|
||||
# Cache successful response
|
||||
self._text_cache[url] = text
|
||||
try:
|
||||
with open(self._cache_path(url) + ".txt", "w", encoding="utf-8") as f:
|
||||
f.write(text)
|
||||
except Exception:
|
||||
pass
|
||||
return text
|
||||
except Exception as e:
|
||||
last_exc = e
|
||||
delay = backoff
|
||||
if self._deadline is not None:
|
||||
remaining = max(0.0, self._deadline - _time.monotonic())
|
||||
delay = min(delay, max(0.0, remaining - 0.05))
|
||||
await asyncio.sleep(
|
||||
max(0.0, delay)
|
||||
+ random.uniform(0.0, delay * 0.25 if delay > 0 else 0.0)
|
||||
)
|
||||
backoff *= 1.8
|
||||
continue
|
||||
raise last_exc if last_exc else RuntimeError("Failed to fetch text")
|
||||
Loading…
Reference in New Issue