|
|
|
@ -3,7 +3,9 @@ SEC Filings Service: index, search, and retrieve SEC filings (8-K, 6-K, 20-F, 40
|
|
|
|
and extract exhibit content (e.g., EX-99.1 press releases).
|
|
|
|
and extract exhibit content (e.g., EX-99.1 press releases).
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
import asyncio
|
|
|
|
import logging
|
|
|
|
import logging
|
|
|
|
|
|
|
|
import time
|
|
|
|
from datetime import datetime, timezone
|
|
|
|
from datetime import datetime, timezone
|
|
|
|
from typing import Dict, List, Optional, Set, Tuple
|
|
|
|
from typing import Dict, List, Optional, Set, Tuple
|
|
|
|
|
|
|
|
|
|
|
|
@ -45,127 +47,131 @@ class SECFilingsService:
|
|
|
|
Returns the number of filings indexed (inserted or updated).
|
|
|
|
Returns the number of filings indexed (inserted or updated).
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
ticker = ticker.upper()
|
|
|
|
ticker = ticker.upper()
|
|
|
|
cik = await self._http.get_company_cik(ticker)
|
|
|
|
self._http.set_deadline(60.0)
|
|
|
|
if not cik:
|
|
|
|
try:
|
|
|
|
raise ValueError(f"Could not find CIK for ticker {ticker}")
|
|
|
|
cik = await self._http.get_company_cik(ticker)
|
|
|
|
|
|
|
|
if not cik:
|
|
|
|
cik_int = int(cik)
|
|
|
|
raise ValueError(f"Could not find CIK for ticker {ticker}")
|
|
|
|
url = f"{self._http.sec_base_data}/submissions/CIK{cik_int:010d}.json"
|
|
|
|
|
|
|
|
data = await self._http.fetch_json(url)
|
|
|
|
cik_int = int(cik)
|
|
|
|
|
|
|
|
url = f"{self._http.sec_base_data}/submissions/CIK{cik_int:010d}.json"
|
|
|
|
# Determine which form types to index
|
|
|
|
data = await self._http.fetch_json(url)
|
|
|
|
target_forms = form_types or self.SUPPORTED_FORM_TYPES
|
|
|
|
|
|
|
|
|
|
|
|
# Determine which form types to index
|
|
|
|
# Collect filings from recent block
|
|
|
|
target_forms = form_types or self.SUPPORTED_FORM_TYPES
|
|
|
|
raw_filings: List[Dict] = []
|
|
|
|
|
|
|
|
|
|
|
|
# Collect filings from recent block
|
|
|
|
def add_from_block(block: dict) -> None:
|
|
|
|
raw_filings: List[Dict] = []
|
|
|
|
forms = block.get("form", [])
|
|
|
|
|
|
|
|
dates = block.get("filingDate", [])
|
|
|
|
def add_from_block(block: dict) -> None:
|
|
|
|
accessions = block.get("accessionNumber", [])
|
|
|
|
forms = block.get("form", [])
|
|
|
|
primary_docs = block.get("primaryDocument", [])
|
|
|
|
dates = block.get("filingDate", [])
|
|
|
|
descriptions = block.get("primaryDocDescription", [])
|
|
|
|
accessions = block.get("accessionNumber", [])
|
|
|
|
accepted_dates = block.get("acceptanceDateTime", [])
|
|
|
|
primary_docs = block.get("primaryDocument", [])
|
|
|
|
for i, (form, dt_str, acc) in enumerate(zip(forms, dates, accessions)):
|
|
|
|
descriptions = block.get("primaryDocDescription", [])
|
|
|
|
if form not in target_forms:
|
|
|
|
accepted_dates = block.get("acceptanceDateTime", [])
|
|
|
|
|
|
|
|
for i, (form, dt_str, acc) in enumerate(zip(forms, dates, accessions)):
|
|
|
|
|
|
|
|
if form not in target_forms:
|
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
pri_doc = primary_docs[i] if i < len(primary_docs) else None
|
|
|
|
|
|
|
|
desc = descriptions[i] if i < len(descriptions) else None
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
|
|
|
filing_date = datetime.strptime(dt_str, "%Y-%m-%d").replace(
|
|
|
|
|
|
|
|
tzinfo=timezone.utc
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
except Exception:
|
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
accepted_at = None
|
|
|
|
|
|
|
|
if i < len(accepted_dates) and accepted_dates[i]:
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
|
|
|
accepted_at = datetime.fromisoformat(
|
|
|
|
|
|
|
|
accepted_dates[i].replace("Z", "+00:00")
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
except Exception:
|
|
|
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
acc_clean = acc.replace("-", "")
|
|
|
|
|
|
|
|
pri_doc_url = None
|
|
|
|
|
|
|
|
if pri_doc:
|
|
|
|
|
|
|
|
pri_doc_url = (
|
|
|
|
|
|
|
|
f"https://www.sec.gov/Archives/edgar/data/{cik_int}/{acc_clean}/{pri_doc}"
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
raw_filings.append({
|
|
|
|
|
|
|
|
"ticker": ticker,
|
|
|
|
|
|
|
|
"cik": cik,
|
|
|
|
|
|
|
|
"accession_number": acc,
|
|
|
|
|
|
|
|
"form_type": form,
|
|
|
|
|
|
|
|
"filing_date": filing_date,
|
|
|
|
|
|
|
|
"accepted_at": accepted_at,
|
|
|
|
|
|
|
|
"primary_document": pri_doc,
|
|
|
|
|
|
|
|
"primary_document_url": pri_doc_url,
|
|
|
|
|
|
|
|
"filing_description": desc,
|
|
|
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
recent = data.get("filings", {}).get("recent", {})
|
|
|
|
|
|
|
|
add_from_block(recent)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Fetch older yearly submission files
|
|
|
|
|
|
|
|
files_meta = data.get("filings", {}).get("files", []) or []
|
|
|
|
|
|
|
|
for meta in files_meta[:6]:
|
|
|
|
|
|
|
|
name = meta.get("name")
|
|
|
|
|
|
|
|
if not name:
|
|
|
|
continue
|
|
|
|
continue
|
|
|
|
pri_doc = primary_docs[i] if i < len(primary_docs) else None
|
|
|
|
older_url = f"{self._http.sec_base_data}/submissions/{name}"
|
|
|
|
desc = descriptions[i] if i < len(descriptions) else None
|
|
|
|
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
filing_date = datetime.strptime(dt_str, "%Y-%m-%d").replace(
|
|
|
|
older = await self._http.fetch_json(older_url)
|
|
|
|
tzinfo=timezone.utc
|
|
|
|
add_from_block(older)
|
|
|
|
)
|
|
|
|
|
|
|
|
except Exception:
|
|
|
|
except Exception:
|
|
|
|
continue
|
|
|
|
continue
|
|
|
|
accepted_at = None
|
|
|
|
|
|
|
|
if i < len(accepted_dates) and accepted_dates[i]:
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
|
|
|
accepted_at = datetime.fromisoformat(
|
|
|
|
|
|
|
|
accepted_dates[i].replace("Z", "+00:00")
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
except Exception:
|
|
|
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
acc_clean = acc.replace("-", "")
|
|
|
|
|
|
|
|
pri_doc_url = None
|
|
|
|
|
|
|
|
if pri_doc:
|
|
|
|
|
|
|
|
pri_doc_url = (
|
|
|
|
|
|
|
|
f"https://www.sec.gov/Archives/edgar/data/{cik_int}/{acc_clean}/{pri_doc}"
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
raw_filings.append({
|
|
|
|
|
|
|
|
"ticker": ticker,
|
|
|
|
|
|
|
|
"cik": cik,
|
|
|
|
|
|
|
|
"accession_number": acc,
|
|
|
|
|
|
|
|
"form_type": form,
|
|
|
|
|
|
|
|
"filing_date": filing_date,
|
|
|
|
|
|
|
|
"accepted_at": accepted_at,
|
|
|
|
|
|
|
|
"primary_document": pri_doc,
|
|
|
|
|
|
|
|
"primary_document_url": pri_doc_url,
|
|
|
|
|
|
|
|
"filing_description": desc,
|
|
|
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
recent = data.get("filings", {}).get("recent", {})
|
|
|
|
if not raw_filings:
|
|
|
|
add_from_block(recent)
|
|
|
|
return 0
|
|
|
|
|
|
|
|
|
|
|
|
# Fetch older yearly submission files
|
|
|
|
# Upsert into DB
|
|
|
|
files_meta = data.get("filings", {}).get("files", []) or []
|
|
|
|
indexed_count = 0
|
|
|
|
for meta in files_meta[:6]:
|
|
|
|
now = datetime.now(timezone.utc)
|
|
|
|
name = meta.get("name")
|
|
|
|
for rf in raw_filings:
|
|
|
|
if not name:
|
|
|
|
acc = rf["accession_number"]
|
|
|
|
continue
|
|
|
|
result = await db.execute(
|
|
|
|
older_url = f"{self._http.sec_base_data}/submissions/{name}"
|
|
|
|
select(SECFiling).where(SECFiling.accession_number == acc)
|
|
|
|
try:
|
|
|
|
|
|
|
|
older = await self._http.fetch_json(older_url)
|
|
|
|
|
|
|
|
add_from_block(older)
|
|
|
|
|
|
|
|
except Exception:
|
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if not raw_filings:
|
|
|
|
|
|
|
|
return 0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Upsert into DB
|
|
|
|
|
|
|
|
indexed_count = 0
|
|
|
|
|
|
|
|
now = datetime.now(timezone.utc)
|
|
|
|
|
|
|
|
for rf in raw_filings:
|
|
|
|
|
|
|
|
acc = rf["accession_number"]
|
|
|
|
|
|
|
|
result = await db.execute(
|
|
|
|
|
|
|
|
select(SECFiling).where(SECFiling.accession_number == acc)
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
existing = result.scalar_one_or_none()
|
|
|
|
|
|
|
|
if existing:
|
|
|
|
|
|
|
|
if force_refresh:
|
|
|
|
|
|
|
|
existing.ticker = rf["ticker"]
|
|
|
|
|
|
|
|
existing.cik = rf["cik"]
|
|
|
|
|
|
|
|
existing.form_type = rf["form_type"]
|
|
|
|
|
|
|
|
existing.filing_date = rf["filing_date"]
|
|
|
|
|
|
|
|
existing.accepted_at = rf["accepted_at"]
|
|
|
|
|
|
|
|
existing.primary_document = rf["primary_document"]
|
|
|
|
|
|
|
|
existing.primary_document_url = rf["primary_document_url"]
|
|
|
|
|
|
|
|
existing.filing_description = rf["filing_description"]
|
|
|
|
|
|
|
|
existing.indexed_at = now
|
|
|
|
|
|
|
|
existing.updated_at = now
|
|
|
|
|
|
|
|
indexed_count += 1
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
filing = SECFiling(
|
|
|
|
|
|
|
|
ticker=rf["ticker"],
|
|
|
|
|
|
|
|
cik=rf["cik"],
|
|
|
|
|
|
|
|
accession_number=rf["accession_number"],
|
|
|
|
|
|
|
|
form_type=rf["form_type"],
|
|
|
|
|
|
|
|
filing_date=rf["filing_date"],
|
|
|
|
|
|
|
|
accepted_at=rf["accepted_at"],
|
|
|
|
|
|
|
|
primary_document=rf["primary_document"],
|
|
|
|
|
|
|
|
primary_document_url=rf["primary_document_url"],
|
|
|
|
|
|
|
|
filing_description=rf["filing_description"],
|
|
|
|
|
|
|
|
indexed_at=now,
|
|
|
|
|
|
|
|
created_at=now,
|
|
|
|
|
|
|
|
updated_at=now,
|
|
|
|
|
|
|
|
)
|
|
|
|
)
|
|
|
|
db.add(filing)
|
|
|
|
existing = result.scalar_one_or_none()
|
|
|
|
indexed_count += 1
|
|
|
|
if existing:
|
|
|
|
|
|
|
|
if force_refresh:
|
|
|
|
|
|
|
|
existing.ticker = rf["ticker"]
|
|
|
|
|
|
|
|
existing.cik = rf["cik"]
|
|
|
|
|
|
|
|
existing.form_type = rf["form_type"]
|
|
|
|
|
|
|
|
existing.filing_date = rf["filing_date"]
|
|
|
|
|
|
|
|
existing.accepted_at = rf["accepted_at"]
|
|
|
|
|
|
|
|
existing.primary_document = rf["primary_document"]
|
|
|
|
|
|
|
|
existing.primary_document_url = rf["primary_document_url"]
|
|
|
|
|
|
|
|
existing.filing_description = rf["filing_description"]
|
|
|
|
|
|
|
|
existing.indexed_at = now
|
|
|
|
|
|
|
|
existing.updated_at = now
|
|
|
|
|
|
|
|
indexed_count += 1
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
filing = SECFiling(
|
|
|
|
|
|
|
|
ticker=rf["ticker"],
|
|
|
|
|
|
|
|
cik=rf["cik"],
|
|
|
|
|
|
|
|
accession_number=rf["accession_number"],
|
|
|
|
|
|
|
|
form_type=rf["form_type"],
|
|
|
|
|
|
|
|
filing_date=rf["filing_date"],
|
|
|
|
|
|
|
|
accepted_at=rf["accepted_at"],
|
|
|
|
|
|
|
|
primary_document=rf["primary_document"],
|
|
|
|
|
|
|
|
primary_document_url=rf["primary_document_url"],
|
|
|
|
|
|
|
|
filing_description=rf["filing_description"],
|
|
|
|
|
|
|
|
indexed_at=now,
|
|
|
|
|
|
|
|
created_at=now,
|
|
|
|
|
|
|
|
updated_at=now,
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
db.add(filing)
|
|
|
|
|
|
|
|
indexed_count += 1
|
|
|
|
|
|
|
|
|
|
|
|
if indexed_count:
|
|
|
|
if indexed_count:
|
|
|
|
await db.commit()
|
|
|
|
await db.commit()
|
|
|
|
logger.info(f"Indexed {indexed_count} filings for {ticker}")
|
|
|
|
logger.info(f"Indexed {indexed_count} filings for {ticker}")
|
|
|
|
return indexed_count
|
|
|
|
return indexed_count
|
|
|
|
|
|
|
|
finally:
|
|
|
|
|
|
|
|
self._http.clear_deadline()
|
|
|
|
|
|
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
|
# ------------------------------------------------------------------
|
|
|
|
# search_filings: query DB with filters
|
|
|
|
# search_filings: query DB with filters
|
|
|
|
@ -235,69 +241,76 @@ class SECFilingsService:
|
|
|
|
self, db: AsyncSession, accession_number: str
|
|
|
|
self, db: AsyncSession, accession_number: str
|
|
|
|
) -> List[Dict]:
|
|
|
|
) -> List[Dict]:
|
|
|
|
"""Get list of documents for a filing. Caches in documents_json column."""
|
|
|
|
"""Get list of documents for a filing. Caches in documents_json column."""
|
|
|
|
result = await db.execute(
|
|
|
|
_owned_deadline = self._http.remaining_time() is None
|
|
|
|
select(SECFiling).where(SECFiling.accession_number == accession_number)
|
|
|
|
if _owned_deadline:
|
|
|
|
)
|
|
|
|
self._http.set_deadline(30.0)
|
|
|
|
filing = result.scalar_one_or_none()
|
|
|
|
try:
|
|
|
|
if not filing:
|
|
|
|
result = await db.execute(
|
|
|
|
raise ValueError(f"Filing {accession_number} not found in database")
|
|
|
|
select(SECFiling).where(SECFiling.accession_number == accession_number)
|
|
|
|
|
|
|
|
)
|
|
|
|
# Return cached documents if available
|
|
|
|
filing = result.scalar_one_or_none()
|
|
|
|
if filing.documents_json:
|
|
|
|
if not filing:
|
|
|
|
return filing.documents_json
|
|
|
|
raise ValueError(f"Filing {accession_number} not found in database")
|
|
|
|
|
|
|
|
|
|
|
|
# Fetch and parse index page
|
|
|
|
# Return cached documents if available
|
|
|
|
cik_int = int("".join(ch for ch in filing.cik if ch.isdigit()))
|
|
|
|
if filing.documents_json:
|
|
|
|
acc_clean = accession_number.replace("-", "")
|
|
|
|
return filing.documents_json
|
|
|
|
index_url = (
|
|
|
|
|
|
|
|
f"https://www.sec.gov/Archives/edgar/data/{cik_int}"
|
|
|
|
# Fetch and parse index page
|
|
|
|
f"/{acc_clean}/{accession_number}-index.htm"
|
|
|
|
cik_int = int("".join(ch for ch in filing.cik if ch.isdigit()))
|
|
|
|
)
|
|
|
|
acc_clean = accession_number.replace("-", "")
|
|
|
|
|
|
|
|
index_url = (
|
|
|
|
|
|
|
|
f"https://www.sec.gov/Archives/edgar/data/{cik_int}"
|
|
|
|
|
|
|
|
f"/{acc_clean}/{accession_number}-index.htm"
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
html = await self._http.fetch_text(index_url)
|
|
|
|
html = await self._http.fetch_text(index_url)
|
|
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
|
|
|
|
|
|
|
|
|
|
base_url = index_url.rsplit("/", 1)[0]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def mk_abs(href: str) -> str:
|
|
|
|
|
|
|
|
if href.startswith("http"):
|
|
|
|
|
|
|
|
return href
|
|
|
|
|
|
|
|
if href.startswith("/"):
|
|
|
|
|
|
|
|
return f"https://www.sec.gov{href}"
|
|
|
|
|
|
|
|
return f"{base_url}/{href}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
documents: List[Dict] = []
|
|
|
|
|
|
|
|
for row in soup.find_all("tr"):
|
|
|
|
|
|
|
|
cells = row.find_all(["td", "th"])
|
|
|
|
|
|
|
|
if len(cells) < 4:
|
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
# Typical columns: Seq, Description, Document, Type, Size
|
|
|
|
|
|
|
|
desc = cells[1].get_text(strip=True) if len(cells) > 1 else ""
|
|
|
|
|
|
|
|
doc_cell = cells[2]
|
|
|
|
|
|
|
|
doc_type = cells[3].get_text(strip=True) if len(cells) > 3 else ""
|
|
|
|
|
|
|
|
size_text = cells[4].get_text(strip=True) if len(cells) > 4 else ""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
a_tag = doc_cell.find("a")
|
|
|
|
|
|
|
|
if not a_tag or not a_tag.get("href"):
|
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
href = a_tag["href"]
|
|
|
|
|
|
|
|
filename = a_tag.get_text(strip=True) or doc_cell.get_text(strip=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
documents.append({
|
|
|
|
|
|
|
|
"type": doc_type,
|
|
|
|
|
|
|
|
"description": desc,
|
|
|
|
|
|
|
|
"filename": filename,
|
|
|
|
|
|
|
|
"url": mk_abs(href),
|
|
|
|
|
|
|
|
"size": size_text,
|
|
|
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Cache to DB
|
|
|
|
base_url = index_url.rsplit("/", 1)[0]
|
|
|
|
if documents:
|
|
|
|
|
|
|
|
filing.documents_json = documents
|
|
|
|
|
|
|
|
filing.updated_at = datetime.now(timezone.utc)
|
|
|
|
|
|
|
|
await db.commit()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return documents
|
|
|
|
def mk_abs(href: str) -> str:
|
|
|
|
|
|
|
|
if href.startswith("http"):
|
|
|
|
|
|
|
|
return href
|
|
|
|
|
|
|
|
if href.startswith("/"):
|
|
|
|
|
|
|
|
return f"https://www.sec.gov{href}"
|
|
|
|
|
|
|
|
return f"{base_url}/{href}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
documents: List[Dict] = []
|
|
|
|
|
|
|
|
for row in soup.find_all("tr"):
|
|
|
|
|
|
|
|
cells = row.find_all(["td", "th"])
|
|
|
|
|
|
|
|
if len(cells) < 4:
|
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
# Typical columns: Seq, Description, Document, Type, Size
|
|
|
|
|
|
|
|
desc = cells[1].get_text(strip=True) if len(cells) > 1 else ""
|
|
|
|
|
|
|
|
doc_cell = cells[2]
|
|
|
|
|
|
|
|
doc_type = cells[3].get_text(strip=True) if len(cells) > 3 else ""
|
|
|
|
|
|
|
|
size_text = cells[4].get_text(strip=True) if len(cells) > 4 else ""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
a_tag = doc_cell.find("a")
|
|
|
|
|
|
|
|
if not a_tag or not a_tag.get("href"):
|
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
href = a_tag["href"]
|
|
|
|
|
|
|
|
filename = a_tag.get_text(strip=True) or doc_cell.get_text(strip=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
documents.append({
|
|
|
|
|
|
|
|
"type": doc_type,
|
|
|
|
|
|
|
|
"description": desc,
|
|
|
|
|
|
|
|
"filename": filename,
|
|
|
|
|
|
|
|
"url": mk_abs(href),
|
|
|
|
|
|
|
|
"size": size_text,
|
|
|
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Cache to DB
|
|
|
|
|
|
|
|
if documents:
|
|
|
|
|
|
|
|
filing.documents_json = documents
|
|
|
|
|
|
|
|
filing.updated_at = datetime.now(timezone.utc)
|
|
|
|
|
|
|
|
await db.commit()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return documents
|
|
|
|
|
|
|
|
finally:
|
|
|
|
|
|
|
|
if _owned_deadline:
|
|
|
|
|
|
|
|
self._http.clear_deadline()
|
|
|
|
|
|
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
|
# ------------------------------------------------------------------
|
|
|
|
# get_exhibit_content: extract exhibit text
|
|
|
|
# get_exhibit_content: extract exhibit text
|
|
|
|
@ -310,56 +323,151 @@ class SECFilingsService:
|
|
|
|
exhibit_type: str = "EX-99.1",
|
|
|
|
exhibit_type: str = "EX-99.1",
|
|
|
|
) -> Dict:
|
|
|
|
) -> Dict:
|
|
|
|
"""Download and return the content of a specific exhibit."""
|
|
|
|
"""Download and return the content of a specific exhibit."""
|
|
|
|
documents = await self.get_filing_documents(db, accession_number)
|
|
|
|
self._http.set_deadline(30.0)
|
|
|
|
if not documents:
|
|
|
|
try:
|
|
|
|
raise ValueError(f"No documents found for filing {accession_number}")
|
|
|
|
documents = await self.get_filing_documents(db, accession_number)
|
|
|
|
|
|
|
|
if not documents:
|
|
|
|
exhibit_type_upper = exhibit_type.upper()
|
|
|
|
raise ValueError(f"No documents found for filing {accession_number}")
|
|
|
|
# Find matching document by type
|
|
|
|
|
|
|
|
target = None
|
|
|
|
exhibit_type_upper = exhibit_type.upper()
|
|
|
|
for doc in documents:
|
|
|
|
# Find matching document by type
|
|
|
|
doc_type = (doc.get("type") or "").upper()
|
|
|
|
target = None
|
|
|
|
if doc_type == exhibit_type_upper:
|
|
|
|
|
|
|
|
target = doc
|
|
|
|
|
|
|
|
break
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Fallback: try matching description
|
|
|
|
|
|
|
|
if not target:
|
|
|
|
|
|
|
|
for doc in documents:
|
|
|
|
for doc in documents:
|
|
|
|
desc = (doc.get("description") or "").upper()
|
|
|
|
doc_type = (doc.get("type") or "").upper()
|
|
|
|
if exhibit_type_upper in desc:
|
|
|
|
if doc_type == exhibit_type_upper:
|
|
|
|
target = doc
|
|
|
|
target = doc
|
|
|
|
break
|
|
|
|
break
|
|
|
|
|
|
|
|
|
|
|
|
if not target:
|
|
|
|
# Fallback: try matching description
|
|
|
|
raise ValueError(
|
|
|
|
if not target:
|
|
|
|
f"Exhibit {exhibit_type} not found in filing {accession_number}. "
|
|
|
|
for doc in documents:
|
|
|
|
f"Available types: {[d.get('type') for d in documents]}"
|
|
|
|
desc = (doc.get("description") or "").upper()
|
|
|
|
)
|
|
|
|
if exhibit_type_upper in desc:
|
|
|
|
|
|
|
|
target = doc
|
|
|
|
|
|
|
|
break
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if not target:
|
|
|
|
|
|
|
|
raise ValueError(
|
|
|
|
|
|
|
|
f"Exhibit {exhibit_type} not found in filing {accession_number}. "
|
|
|
|
|
|
|
|
f"Available types: {[d.get('type') for d in documents]}"
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
url = target["url"]
|
|
|
|
url = target["url"]
|
|
|
|
content = await self._http.fetch_text(url)
|
|
|
|
content = await self._http.fetch_text(url)
|
|
|
|
|
|
|
|
|
|
|
|
if len(content) > MAX_EXHIBIT_SIZE:
|
|
|
|
if len(content) > MAX_EXHIBIT_SIZE:
|
|
|
|
raise ValueError(
|
|
|
|
raise ValueError(
|
|
|
|
f"Exhibit content exceeds size limit ({len(content)} bytes > {MAX_EXHIBIT_SIZE} bytes)"
|
|
|
|
f"Exhibit content exceeds size limit ({len(content)} bytes > {MAX_EXHIBIT_SIZE} bytes)"
|
|
|
|
)
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Determine content type
|
|
|
|
|
|
|
|
filename = target.get("filename", "")
|
|
|
|
|
|
|
|
if filename.lower().endswith(".htm") or filename.lower().endswith(".html"):
|
|
|
|
|
|
|
|
content_type = "text/html"
|
|
|
|
|
|
|
|
elif filename.lower().endswith(".xml"):
|
|
|
|
|
|
|
|
content_type = "application/xml"
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
content_type = "text/plain"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return {
|
|
|
|
|
|
|
|
"content": content,
|
|
|
|
|
|
|
|
"content_type": content_type,
|
|
|
|
|
|
|
|
"filename": filename,
|
|
|
|
|
|
|
|
"url": url,
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
finally:
|
|
|
|
|
|
|
|
self._http.clear_deadline()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
|
|
|
# search_filings_bulk: bulk filing search for multiple tickers
|
|
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async def search_filings_bulk(
|
|
|
|
|
|
|
|
self,
|
|
|
|
|
|
|
|
db: AsyncSession,
|
|
|
|
|
|
|
|
tickers: List[str],
|
|
|
|
|
|
|
|
form_types: Optional[Set[str]] = None,
|
|
|
|
|
|
|
|
start_date: Optional[datetime] = None,
|
|
|
|
|
|
|
|
end_date: Optional[datetime] = None,
|
|
|
|
|
|
|
|
limit_per_ticker: int = 20,
|
|
|
|
|
|
|
|
) -> List[Dict]:
|
|
|
|
|
|
|
|
"""Bulk search filings for multiple tickers. Auto-indexes missing tickers in parallel."""
|
|
|
|
|
|
|
|
t0 = time.monotonic()
|
|
|
|
|
|
|
|
tickers = [t.upper() for t in tickers]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Identify which tickers already have indexed filings
|
|
|
|
|
|
|
|
conditions = [SECFiling.ticker.in_(tickers)]
|
|
|
|
|
|
|
|
if form_types:
|
|
|
|
|
|
|
|
conditions.append(SECFiling.form_type.in_(form_types))
|
|
|
|
|
|
|
|
count_result = await db.execute(
|
|
|
|
|
|
|
|
select(SECFiling.ticker, func.count().label("cnt"))
|
|
|
|
|
|
|
|
.where(and_(*conditions))
|
|
|
|
|
|
|
|
.group_by(SECFiling.ticker)
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
indexed_counts = {row.ticker: row.cnt for row in count_result}
|
|
|
|
|
|
|
|
missing = [t for t in tickers if indexed_counts.get(t, 0) == 0]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Index missing tickers in parallel (Semaphore(4) to not overwhelm SEC)
|
|
|
|
|
|
|
|
if missing:
|
|
|
|
|
|
|
|
sem = asyncio.Semaphore(4)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async def _index_one(ticker: str) -> None:
|
|
|
|
|
|
|
|
async with sem:
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
|
|
|
await asyncio.wait_for(
|
|
|
|
|
|
|
|
self.index_filings(db, ticker, form_types=form_types),
|
|
|
|
|
|
|
|
timeout=60,
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
except Exception as e:
|
|
|
|
|
|
|
|
logger.warning(f"Bulk index failed for {ticker}: {e}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
await asyncio.gather(*[_index_one(t) for t in missing])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Fetch results for all tickers from DB
|
|
|
|
|
|
|
|
results = []
|
|
|
|
|
|
|
|
for ticker in tickers:
|
|
|
|
|
|
|
|
ticker_conditions = [SECFiling.ticker == ticker]
|
|
|
|
|
|
|
|
if form_types:
|
|
|
|
|
|
|
|
ticker_conditions.append(SECFiling.form_type.in_(form_types))
|
|
|
|
|
|
|
|
if start_date:
|
|
|
|
|
|
|
|
ticker_conditions.append(SECFiling.filing_date >= start_date)
|
|
|
|
|
|
|
|
if end_date:
|
|
|
|
|
|
|
|
ticker_conditions.append(SECFiling.filing_date <= end_date)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
|
|
|
count_res = await db.execute(
|
|
|
|
|
|
|
|
select(func.count())
|
|
|
|
|
|
|
|
.select_from(SECFiling)
|
|
|
|
|
|
|
|
.where(and_(*ticker_conditions))
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
total = count_res.scalar() or 0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
rows = await db.execute(
|
|
|
|
|
|
|
|
select(SECFiling)
|
|
|
|
|
|
|
|
.where(and_(*ticker_conditions))
|
|
|
|
|
|
|
|
.order_by(SECFiling.filing_date.desc())
|
|
|
|
|
|
|
|
.limit(limit_per_ticker)
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
filings = rows.scalars().all()
|
|
|
|
|
|
|
|
results.append({
|
|
|
|
|
|
|
|
"ticker": ticker,
|
|
|
|
|
|
|
|
"success": True,
|
|
|
|
|
|
|
|
"filings": list(filings),
|
|
|
|
|
|
|
|
"total_count": total,
|
|
|
|
|
|
|
|
"error": None,
|
|
|
|
|
|
|
|
})
|
|
|
|
|
|
|
|
except Exception as e:
|
|
|
|
|
|
|
|
results.append({
|
|
|
|
|
|
|
|
"ticker": ticker,
|
|
|
|
|
|
|
|
"success": False,
|
|
|
|
|
|
|
|
"filings": [],
|
|
|
|
|
|
|
|
"total_count": 0,
|
|
|
|
|
|
|
|
"error": str(e),
|
|
|
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
|
|
# Determine content type
|
|
|
|
elapsed = time.monotonic() - t0
|
|
|
|
filename = target.get("filename", "")
|
|
|
|
return results, elapsed
|
|
|
|
if filename.lower().endswith(".htm") or filename.lower().endswith(".html"):
|
|
|
|
|
|
|
|
content_type = "text/html"
|
|
|
|
|
|
|
|
elif filename.lower().endswith(".xml"):
|
|
|
|
|
|
|
|
content_type = "application/xml"
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
content_type = "text/plain"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return {
|
|
|
|
|
|
|
|
"content": content,
|
|
|
|
|
|
|
|
"content_type": content_type,
|
|
|
|
|
|
|
|
"filename": filename,
|
|
|
|
|
|
|
|
"url": url,
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Module-level singleton
|
|
|
|
# Module-level singleton
|
|
|
|
|