""" SEC Filings Service: index, search, and retrieve SEC filings (8-K, 6-K, 20-F, 40-F, etc.) and extract exhibit content (e.g., EX-99.1 press releases). """ import logging from datetime import datetime, timezone from typing import Dict, List, Optional, Set, Tuple from bs4 import BeautifulSoup from sqlalchemy import select, and_, func from sqlalchemy.ext.asyncio import AsyncSession from app.models.filing import SECFiling from app.services.sec_http_client import SECHttpClient logger = logging.getLogger(__name__) # Maximum exhibit content size (1 MB) MAX_EXHIBIT_SIZE = 1 * 1024 * 1024 class SECFilingsService: SUPPORTED_FORM_TYPES: Set[str] = { "8-K", "6-K", "20-F", "40-F", "10-K", "10-Q", "8-K/A", "6-K/A", "20-F/A", "40-F/A", } def __init__(self): self._http = SECHttpClient("Stock Oracle Filings Service") # ------------------------------------------------------------------ # index_filings: fetch from SEC submissions and upsert to DB # ------------------------------------------------------------------ async def index_filings( self, db: AsyncSession, ticker: str, form_types: Optional[Set[str]] = None, force_refresh: bool = False, ) -> int: """Index filings from SEC submissions endpoint into sec_filings table. Returns the number of filings indexed (inserted or updated). """ ticker = ticker.upper() cik = await self._http.get_company_cik(ticker) if not cik: raise ValueError(f"Could not find CIK for ticker {ticker}") cik_int = int(cik) url = f"{self._http.sec_base_data}/submissions/CIK{cik_int:010d}.json" data = await self._http.fetch_json(url) # Determine which form types to index target_forms = form_types or self.SUPPORTED_FORM_TYPES # Collect filings from recent block raw_filings: List[Dict] = [] def add_from_block(block: dict) -> None: forms = block.get("form", []) dates = block.get("filingDate", []) accessions = block.get("accessionNumber", []) primary_docs = block.get("primaryDocument", []) descriptions = block.get("primaryDocDescription", []) for i, (form, dt_str, acc) in enumerate(zip(forms, dates, accessions)): if form not in target_forms: continue pri_doc = primary_docs[i] if i < len(primary_docs) else None desc = descriptions[i] if i < len(descriptions) else None try: filing_date = datetime.strptime(dt_str, "%Y-%m-%d").replace( tzinfo=timezone.utc ) except Exception: continue acc_clean = acc.replace("-", "") pri_doc_url = None if pri_doc: pri_doc_url = ( f"https://www.sec.gov/Archives/edgar/data/{cik_int}/{acc_clean}/{pri_doc}" ) raw_filings.append({ "ticker": ticker, "cik": cik, "accession_number": acc, "form_type": form, "filing_date": filing_date, "primary_document": pri_doc, "primary_document_url": pri_doc_url, "filing_description": desc, }) recent = data.get("filings", {}).get("recent", {}) add_from_block(recent) # Fetch older yearly submission files files_meta = data.get("filings", {}).get("files", []) or [] for meta in files_meta[:6]: name = meta.get("name") if not name: continue older_url = f"{self._http.sec_base_data}/submissions/{name}" try: older = await self._http.fetch_json(older_url) add_from_block(older) except Exception: continue if not raw_filings: return 0 # Upsert into DB indexed_count = 0 now = datetime.now(timezone.utc) for rf in raw_filings: acc = rf["accession_number"] result = await db.execute( select(SECFiling).where(SECFiling.accession_number == acc) ) existing = result.scalar_one_or_none() if existing: if force_refresh: existing.ticker = rf["ticker"] existing.cik = rf["cik"] existing.form_type = rf["form_type"] existing.filing_date = rf["filing_date"] existing.primary_document = rf["primary_document"] existing.primary_document_url = rf["primary_document_url"] existing.filing_description = rf["filing_description"] existing.indexed_at = now existing.updated_at = now indexed_count += 1 else: filing = SECFiling( ticker=rf["ticker"], cik=rf["cik"], accession_number=rf["accession_number"], form_type=rf["form_type"], filing_date=rf["filing_date"], primary_document=rf["primary_document"], primary_document_url=rf["primary_document_url"], filing_description=rf["filing_description"], indexed_at=now, created_at=now, updated_at=now, ) db.add(filing) indexed_count += 1 if indexed_count: await db.commit() logger.info(f"Indexed {indexed_count} filings for {ticker}") return indexed_count # ------------------------------------------------------------------ # search_filings: query DB with filters # ------------------------------------------------------------------ async def search_filings( self, db: AsyncSession, ticker: str, form_types: Optional[Set[str]] = None, start_date: Optional[datetime] = None, end_date: Optional[datetime] = None, limit: int = 20, offset: int = 0, ) -> Tuple[List[SECFiling], int]: """Search indexed filings. Auto-indexes if none found.""" ticker = ticker.upper() conditions = [SECFiling.ticker == ticker] if form_types: conditions.append(SECFiling.form_type.in_(form_types)) if start_date: conditions.append(SECFiling.filing_date >= start_date) if end_date: conditions.append(SECFiling.filing_date <= end_date) where_clause = and_(*conditions) # Count count_result = await db.execute( select(func.count()).select_from(SECFiling).where(where_clause) ) total_count = count_result.scalar() or 0 # If no results, try indexing first if total_count == 0: try: await self.index_filings(db, ticker, form_types=form_types) except Exception as e: logger.warning(f"Auto-indexing failed for {ticker}: {e}") return [], 0 # Re-count after indexing count_result = await db.execute( select(func.count()).select_from(SECFiling).where(where_clause) ) total_count = count_result.scalar() or 0 if total_count == 0: return [], 0 # Query with pagination result = await db.execute( select(SECFiling) .where(where_clause) .order_by(SECFiling.filing_date.desc()) .limit(limit) .offset(offset) ) filings = result.scalars().all() return list(filings), total_count # ------------------------------------------------------------------ # get_filing_documents: list all documents in a filing # ------------------------------------------------------------------ async def get_filing_documents( self, db: AsyncSession, accession_number: str ) -> List[Dict]: """Get list of documents for a filing. Caches in documents_json column.""" result = await db.execute( select(SECFiling).where(SECFiling.accession_number == accession_number) ) filing = result.scalar_one_or_none() if not filing: raise ValueError(f"Filing {accession_number} not found in database") # Return cached documents if available if filing.documents_json: return filing.documents_json # Fetch and parse index page cik_int = int("".join(ch for ch in filing.cik if ch.isdigit())) acc_clean = accession_number.replace("-", "") index_url = ( f"https://www.sec.gov/Archives/edgar/data/{cik_int}" f"/{acc_clean}/{accession_number}-index.htm" ) html = await self._http.fetch_text(index_url) soup = BeautifulSoup(html, "html.parser") base_url = index_url.rsplit("/", 1)[0] def mk_abs(href: str) -> str: if href.startswith("http"): return href if href.startswith("/"): return f"https://www.sec.gov{href}" return f"{base_url}/{href}" documents: List[Dict] = [] for row in soup.find_all("tr"): cells = row.find_all(["td", "th"]) if len(cells) < 4: continue # Typical columns: Seq, Description, Document, Type, Size desc = cells[1].get_text(strip=True) if len(cells) > 1 else "" doc_cell = cells[2] doc_type = cells[3].get_text(strip=True) if len(cells) > 3 else "" size_text = cells[4].get_text(strip=True) if len(cells) > 4 else "" a_tag = doc_cell.find("a") if not a_tag or not a_tag.get("href"): continue href = a_tag["href"] filename = a_tag.get_text(strip=True) or doc_cell.get_text(strip=True) documents.append({ "type": doc_type, "description": desc, "filename": filename, "url": mk_abs(href), "size": size_text, }) # Cache to DB if documents: filing.documents_json = documents filing.updated_at = datetime.now(timezone.utc) await db.commit() return documents # ------------------------------------------------------------------ # get_exhibit_content: extract exhibit text # ------------------------------------------------------------------ async def get_exhibit_content( self, db: AsyncSession, accession_number: str, exhibit_type: str = "EX-99.1", ) -> Dict: """Download and return the content of a specific exhibit.""" documents = await self.get_filing_documents(db, accession_number) if not documents: raise ValueError(f"No documents found for filing {accession_number}") exhibit_type_upper = exhibit_type.upper() # Find matching document by type target = None for doc in documents: doc_type = (doc.get("type") or "").upper() if doc_type == exhibit_type_upper: target = doc break # Fallback: try matching description if not target: for doc in documents: desc = (doc.get("description") or "").upper() if exhibit_type_upper in desc: target = doc break if not target: raise ValueError( f"Exhibit {exhibit_type} not found in filing {accession_number}. " f"Available types: {[d.get('type') for d in documents]}" ) url = target["url"] content = await self._http.fetch_text(url) if len(content) > MAX_EXHIBIT_SIZE: raise ValueError( f"Exhibit content exceeds size limit ({len(content)} bytes > {MAX_EXHIBIT_SIZE} bytes)" ) # Determine content type filename = target.get("filename", "") if filename.lower().endswith(".htm") or filename.lower().endswith(".html"): content_type = "text/html" elif filename.lower().endswith(".xml"): content_type = "application/xml" else: content_type = "text/plain" return { "content": content, "content_type": content_type, "filename": filename, "url": url, } # Module-level singleton sec_filings_service = SECFilingsService()