""" SEC EDGAR Direct API Service Fetches real financial data directly from SEC EDGAR without dependencies """ import json from datetime import datetime, timezone, timedelta from typing import Dict, List, Optional, Tuple, Any import logging from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy import select, and_ from app.models.financial import Company, FinancialData, CalculatedMetrics, PriceData from app.schemas.financial import DataSource from app.services.sec_http_client import SECHttpClient logger = logging.getLogger(__name__) class SECEdgarService: """Direct SEC EDGAR API service for financial data""" def __init__(self): self._http = SECHttpClient("Stock Oracle SEC Service") async def get_company_cik(self, ticker: str) -> Optional[str]: """Get company CIK from SEC ticker mapping""" return await self._http.get_company_cik(ticker) async def get_company_facts(self, cik: str) -> Optional[Dict]: """Get company facts from SEC EDGAR API""" try: url = f"{self._http.sec_base_data}/api/xbrl/companyfacts/CIK{cik.zfill(10)}.json" data = await self._http.fetch_json(url) logger.info(f"Successfully fetched SEC facts for CIK {cik}") return data except Exception as e: if "404" in str(e): logger.warning(f"No SEC data found for CIK {cik}") return None logger.error(f"Error fetching company facts for CIK {cik}: {e}") return None def extract_financial_data(self, facts_data: Dict, start_date: datetime, end_date: datetime) -> List[Dict]: """Extract financial data from SEC facts""" try: if not facts_data or 'facts' not in facts_data: return [] facts = facts_data['facts'] financial_records = [] # Common XBRL concepts mapping (without namespace prefix - it's already in the structure) concept_mapping = { # Revenue concepts 'Revenues': 'revenue', 'RevenueFromContractWithCustomerExcludingAssessedTax': 'revenue', 'SalesRevenueNet': 'revenue', # Income concepts 'OperatingIncomeLoss': 'operating_income', 'NetIncomeLoss': 'net_income', 'GrossProfit': 'gross_profit', # Balance sheet concepts 'Assets': 'total_assets', 'StockholdersEquity': 'total_equity', 'LiabilitiesAndStockholdersEquity': 'total_assets', # Alternative for total assets 'Liabilities': 'total_debt', 'CashAndCashEquivalentsAtCarryingValue': 'cash', 'CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalents': 'cash', # Share data 'CommonStockSharesOutstanding': 'shares_outstanding', 'WeightedAverageNumberOfSharesOutstandingBasic': 'shares_outstanding', 'WeightedAverageNumberOfDilutedSharesOutstanding': 'shares_outstanding', # Cash flow concepts 'NetCashProvidedByUsedInOperatingActivities': 'operating_cash_flow', 'PaymentsToAcquirePropertyPlantAndEquipment': 'capex' } # Collect all quarterly and annual data points data_points = {} # Access us-gaap namespace us_gaap_facts = facts.get('us-gaap', {}) for concept, field_name in concept_mapping.items(): if concept in us_gaap_facts: units = us_gaap_facts[concept].get('units', {}) # Try USD first, then shares for share counts unit_key = 'USD' if 'USD' in units else ('shares' if 'shares' in units else None) if unit_key and unit_key in units: for entry in units[unit_key]: # Get the period end date end = entry.get('end') if not end: continue try: # Handle date format like '2016-09-24' if 'T' not in end and 'Z' not in end: period_date = datetime.strptime(end, '%Y-%m-%d') period_date = period_date.replace(tzinfo=timezone.utc) else: period_date = datetime.fromisoformat(end.replace('Z', '+00:00')) except Exception as e: logger.warning(f"Could not parse date {end}: {e}") continue # Check if within date range if period_date < start_date or period_date > end_date: continue # Get period info form = entry.get('form', '') filing_date = entry.get('filed', '') value = entry.get('val') if value is None: continue # Create period key (quarter end date) period_key = period_date.strftime('%Y-%m-%d') if period_key not in data_points: data_points[period_key] = { 'period_date': period_date, 'form': form, 'filing_date': filing_date, 'period_type': 'quarterly' if form == '10-Q' else 'annual' } # Store the value data_points[period_key][field_name] = float(value) # Convert to financial records for period_key, data in data_points.items(): if len(data) > 4: # Must have more than just metadata financial_records.append(data) # Sort by period date financial_records.sort(key=lambda x: x['period_date']) logger.info(f"Extracted {len(financial_records)} financial periods from SEC data") return financial_records except Exception as e: logger.error(f"Error extracting financial data: {e}") return [] async def get_financial_data( self, db: AsyncSession, ticker: str, start_date: datetime, end_date: datetime, force_refresh: bool = False ) -> List[FinancialData]: """Get financial data for a company from SEC EDGAR""" ticker = ticker.upper() # Check if we already have data if not force_refresh: existing_result = await db.execute( select(FinancialData).where( and_( FinancialData.ticker == ticker, FinancialData.period_date >= start_date, FinancialData.period_date <= end_date, FinancialData.data_source == DataSource.SEC_EDGAR.value, FinancialData.is_estimated == False ) ).order_by(FinancialData.period_date) ) existing_data = existing_result.scalars().all() if existing_data: logger.info(f"Found {len(existing_data)} existing SEC records for {ticker}") return existing_data # Get company CIK cik = await self.get_company_cik(ticker) if not cik: logger.error(f"Could not find CIK for ticker {ticker}") return [] # Get company facts from SEC facts_data = await self.get_company_facts(cik) if not facts_data: logger.error(f"Could not fetch SEC facts for {ticker} (CIK: {cik})") return [] # Extract financial data financial_periods = self.extract_financial_data(facts_data, start_date, end_date) if not financial_periods: logger.warning(f"No financial data extracted for {ticker}") return [] # Convert to database records financial_records = [] for period_data in financial_periods: try: # Check if record already exists period_date = period_data['period_date'] period_type = period_data.get('period_type', 'quarterly') existing_result = await db.execute( select(FinancialData).where( and_( FinancialData.ticker == ticker, FinancialData.period_date == period_date, FinancialData.period_type == period_type ) ) ) existing_record = existing_result.scalar_one_or_none() if existing_record and not force_refresh: financial_records.append(existing_record) continue # Calculate EPS if we have net income and shares eps = None net_income = period_data.get('net_income') shares_outstanding = period_data.get('shares_outstanding') if net_income and shares_outstanding and shares_outstanding > 0: eps = net_income / shares_outstanding # Calculate free cash flow free_cash_flow = None operating_cash_flow = period_data.get('operating_cash_flow') capex = period_data.get('capex') if operating_cash_flow and capex: free_cash_flow = operating_cash_flow - abs(capex) # capex is usually negative if existing_record: # Update existing record existing_record.revenue = period_data.get('revenue') existing_record.gross_profit = period_data.get('gross_profit') existing_record.operating_income = period_data.get('operating_income') existing_record.net_income = net_income existing_record.eps = eps existing_record.total_assets = period_data.get('total_assets') existing_record.total_equity = period_data.get('total_equity') existing_record.total_debt = period_data.get('total_debt') existing_record.cash = period_data.get('cash') existing_record.shares_outstanding = shares_outstanding existing_record.operating_cash_flow = operating_cash_flow existing_record.free_cash_flow = free_cash_flow existing_record.capex = abs(capex) if capex else None existing_record.data_source = DataSource.SEC_EDGAR.value existing_record.is_estimated = False existing_record.updated_at = datetime.now(timezone.utc) financial_records.append(existing_record) else: # Create new record filing_type = "10-K" if period_type == "annual" else "10-Q" financial_record = FinancialData( ticker=ticker, period_date=period_date, period_type=period_type, filing_type=filing_type, revenue=period_data.get('revenue'), gross_profit=period_data.get('gross_profit'), operating_income=period_data.get('operating_income'), net_income=net_income, eps=eps, total_assets=period_data.get('total_assets'), total_equity=period_data.get('total_equity'), total_debt=period_data.get('total_debt'), cash=period_data.get('cash'), shares_outstanding=shares_outstanding, operating_cash_flow=operating_cash_flow, free_cash_flow=free_cash_flow, capex=abs(capex) if capex else None, data_source=DataSource.SEC_EDGAR.value, is_estimated=False, created_at=datetime.now(timezone.utc), updated_at=datetime.now(timezone.utc) ) db.add(financial_record) financial_records.append(financial_record) logger.info(f"Processed SEC data for {ticker} {period_date.date()}: Revenue=${period_data.get('revenue', 0):,.0f}") except Exception as e: logger.error(f"Error processing period data for {ticker}: {e}") continue if financial_records: await db.commit() # Refresh all records to get IDs for record in financial_records: if record.id is None: # Only refresh new records await db.refresh(record) logger.info(f"Successfully fetched {len(financial_records)} SEC financial records for {ticker}") return financial_records async def get_company_info(self, ticker: str) -> Dict[str, Any]: """Get company information from SEC""" try: cik = await self.get_company_cik(ticker) if not cik: return self._get_fallback_company_info(ticker) facts_data = await self.get_company_facts(cik) if not facts_data: return self._get_fallback_company_info(ticker) entity_info = facts_data.get('entityName', ticker) return { 'name': entity_info, 'cik': cik, 'sector': 'Technology', # SEC doesn't provide sector info directly 'industry': 'Software', 'business_description': f'{entity_info} - SEC registered company' } except Exception as e: logger.error(f"Error fetching SEC company info for {ticker}: {e}") return self._get_fallback_company_info(ticker) def _get_fallback_company_info(self, ticker: str) -> Dict[str, Any]: """Fallback company info""" return { 'name': f'{ticker} Corporation', 'cik': f'000{hash(ticker) % 1000000:06d}', 'sector': 'Technology', 'industry': 'Software', 'business_description': f'{ticker} technology company' }