stock-oracle/app/services/sec_edgar_service.py

"""
SEC EDGAR Direct API Service
Fetches real financial data directly from SEC EDGAR without dependencies
"""

import json
from datetime import datetime, timezone, timedelta
from typing import Dict, List, Optional, Tuple, Any
import logging
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, and_

from app.models.financial import Company, FinancialData, CalculatedMetrics, PriceData
from app.schemas.financial import DataSource
from app.services.sec_http_client import SECHttpClient

logger = logging.getLogger(__name__)

class SECEdgarService:
    """Direct SEC EDGAR API service for financial data"""

    def __init__(self):
        self._http = SECHttpClient("Stock Oracle SEC Service")

    async def get_company_cik(self, ticker: str) -> Optional[str]:
        """Get company CIK from SEC ticker mapping"""
        return await self._http.get_company_cik(ticker)

    async def get_company_facts(self, cik: str) -> Optional[Dict]:
        """Get company facts from SEC EDGAR API"""
        try:
            url = f"{self._http.sec_base_data}/api/xbrl/companyfacts/CIK{cik.zfill(10)}.json"
            data = await self._http.fetch_json(url)
            logger.info(f"Successfully fetched SEC facts for CIK {cik}")
            return data
        except Exception as e:
            if "404" in str(e):
                logger.warning(f"No SEC data found for CIK {cik}")
                return None
            logger.error(f"Error fetching company facts for CIK {cik}: {e}")
            return None
    
    def extract_financial_data(self, facts_data: Dict, start_date: datetime, end_date: datetime) -> List[Dict]:
        """Extract financial data from SEC facts"""
        try:
            if not facts_data or 'facts' not in facts_data:
                return []
            
            facts = facts_data['facts']
            financial_records = []
            
            # Common XBRL concepts mapping (without namespace prefix - it's already in the structure)
            concept_mapping = {
                # Revenue concepts
                'Revenues': 'revenue',
                'RevenueFromContractWithCustomerExcludingAssessedTax': 'revenue',
                'SalesRevenueNet': 'revenue',
                # Income concepts  
                'OperatingIncomeLoss': 'operating_income',
                'NetIncomeLoss': 'net_income',
                'GrossProfit': 'gross_profit',
                # Balance sheet concepts
                'Assets': 'total_assets',
                'StockholdersEquity': 'total_equity',
                'LiabilitiesAndStockholdersEquity': 'total_assets',  # Alternative for total assets
                'Liabilities': 'total_debt',
                'CashAndCashEquivalentsAtCarryingValue': 'cash',
                'CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalents': 'cash',
                # Share data
                'CommonStockSharesOutstanding': 'shares_outstanding',
                'WeightedAverageNumberOfSharesOutstandingBasic': 'shares_outstanding',
                'WeightedAverageNumberOfDilutedSharesOutstanding': 'shares_outstanding',
                # Cash flow concepts
                'NetCashProvidedByUsedInOperatingActivities': 'operating_cash_flow',
                'PaymentsToAcquirePropertyPlantAndEquipment': 'capex'
            }
            
            # Collect all quarterly and annual data points
            data_points = {}
            
            # Access us-gaap namespace
            us_gaap_facts = facts.get('us-gaap', {})
            
            for concept, field_name in concept_mapping.items():
                if concept in us_gaap_facts:
                    units = us_gaap_facts[concept].get('units', {})
                    
                    # Try USD first, then shares for share counts
                    unit_key = 'USD' if 'USD' in units else ('shares' if 'shares' in units else None)
                    
                    if unit_key and unit_key in units:
                        for entry in units[unit_key]:
                            # Get the period end date
                            end = entry.get('end')
                            if not end:
                                continue
                                
                            try:
                                # Handle date format like '2016-09-24'
                                if 'T' not in end and 'Z' not in end:
                                    period_date = datetime.strptime(end, '%Y-%m-%d')
                                    period_date = period_date.replace(tzinfo=timezone.utc)
                                else:
                                    period_date = datetime.fromisoformat(end.replace('Z', '+00:00'))
                            except Exception as e:
                                logger.warning(f"Could not parse date {end}: {e}")
                                continue
                            
                            # Check if within date range
                            if period_date < start_date or period_date > end_date:
                                continue
                            
                            # Get period info
                            form = entry.get('form', '')
                            filing_date = entry.get('filed', '')
                            value = entry.get('val')
                            
                            if value is None:
                                continue
                            
                            # Create period key (quarter end date)
                            period_key = period_date.strftime('%Y-%m-%d')
                            
                            if period_key not in data_points:
                                data_points[period_key] = {
                                    'period_date': period_date,
                                    'form': form,
                                    'filing_date': filing_date,
                                    'period_type': 'quarterly' if form == '10-Q' else 'annual'
                                }
                            
                            # Store the value
                            data_points[period_key][field_name] = float(value)
            
            # Convert to financial records
            for period_key, data in data_points.items():
                if len(data) > 4:  # Must have more than just metadata
                    financial_records.append(data)
            
            # Sort by period date
            financial_records.sort(key=lambda x: x['period_date'])
            
            logger.info(f"Extracted {len(financial_records)} financial periods from SEC data")
            return financial_records
            
        except Exception as e:
            logger.error(f"Error extracting financial data: {e}")
            return []
    
    async def get_financial_data(
        self,
        db: AsyncSession,
        ticker: str,
        start_date: datetime,
        end_date: datetime,
        force_refresh: bool = False
    ) -> List[FinancialData]:
        """Get financial data for a company from SEC EDGAR"""
        
        ticker = ticker.upper()
        
        # Check if we already have data
        if not force_refresh:
            existing_result = await db.execute(
                select(FinancialData).where(
                    and_(
                        FinancialData.ticker == ticker,
                        FinancialData.period_date >= start_date,
                        FinancialData.period_date <= end_date,
                        FinancialData.data_source == DataSource.SEC_EDGAR.value,
                        FinancialData.is_estimated == False
                    )
                ).order_by(FinancialData.period_date)
            )
            existing_data = existing_result.scalars().all()
            
            if existing_data:
                logger.info(f"Found {len(existing_data)} existing SEC records for {ticker}")
                return existing_data
        
        # Get company CIK
        cik = await self.get_company_cik(ticker)
        if not cik:
            logger.error(f"Could not find CIK for ticker {ticker}")
            return []
        
        # Get company facts from SEC
        facts_data = await self.get_company_facts(cik)
        if not facts_data:
            logger.error(f"Could not fetch SEC facts for {ticker} (CIK: {cik})")
            return []
        
        # Extract financial data
        financial_periods = self.extract_financial_data(facts_data, start_date, end_date)
        
        if not financial_periods:
            logger.warning(f"No financial data extracted for {ticker}")
            return []
        
        # Convert to database records
        financial_records = []
        
        for period_data in financial_periods:
            try:
                # Check if record already exists
                period_date = period_data['period_date']
                period_type = period_data.get('period_type', 'quarterly')
                
                existing_result = await db.execute(
                    select(FinancialData).where(
                        and_(
                            FinancialData.ticker == ticker,
                            FinancialData.period_date == period_date,
                            FinancialData.period_type == period_type
                        )
                    )
                )
                existing_record = existing_result.scalar_one_or_none()
                
                if existing_record and not force_refresh:
                    financial_records.append(existing_record)
                    continue
                
                # Calculate EPS if we have net income and shares
                eps = None
                net_income = period_data.get('net_income')
                shares_outstanding = period_data.get('shares_outstanding')
                if net_income and shares_outstanding and shares_outstanding > 0:
                    eps = net_income / shares_outstanding
                
                # Calculate free cash flow
                free_cash_flow = None
                operating_cash_flow = period_data.get('operating_cash_flow')
                capex = period_data.get('capex')
                if operating_cash_flow and capex:
                    free_cash_flow = operating_cash_flow - abs(capex)  # capex is usually negative
                
                if existing_record:
                    # Update existing record
                    existing_record.revenue = period_data.get('revenue')
                    existing_record.gross_profit = period_data.get('gross_profit')
                    existing_record.operating_income = period_data.get('operating_income')
                    existing_record.net_income = net_income
                    existing_record.eps = eps
                    existing_record.total_assets = period_data.get('total_assets')
                    existing_record.total_equity = period_data.get('total_equity')
                    existing_record.total_debt = period_data.get('total_debt')
                    existing_record.cash = period_data.get('cash')
                    existing_record.shares_outstanding = shares_outstanding
                    existing_record.operating_cash_flow = operating_cash_flow
                    existing_record.free_cash_flow = free_cash_flow
                    existing_record.capex = abs(capex) if capex else None
                    existing_record.data_source = DataSource.SEC_EDGAR.value
                    existing_record.is_estimated = False
                    existing_record.updated_at = datetime.now(timezone.utc)
                    
                    financial_records.append(existing_record)
                else:
                    # Create new record
                    filing_type = "10-K" if period_type == "annual" else "10-Q"
                    
                    financial_record = FinancialData(
                        ticker=ticker,
                        period_date=period_date,
                        period_type=period_type,
                        filing_type=filing_type,
                        revenue=period_data.get('revenue'),
                        gross_profit=period_data.get('gross_profit'),
                        operating_income=period_data.get('operating_income'),
                        net_income=net_income,
                        eps=eps,
                        total_assets=period_data.get('total_assets'),
                        total_equity=period_data.get('total_equity'),
                        total_debt=period_data.get('total_debt'),
                        cash=period_data.get('cash'),
                        shares_outstanding=shares_outstanding,
                        operating_cash_flow=operating_cash_flow,
                        free_cash_flow=free_cash_flow,
                        capex=abs(capex) if capex else None,
                        data_source=DataSource.SEC_EDGAR.value,
                        is_estimated=False,
                        created_at=datetime.now(timezone.utc),
                        updated_at=datetime.now(timezone.utc)
                    )
                    
                    db.add(financial_record)
                    financial_records.append(financial_record)
                
                logger.info(f"Processed SEC data for {ticker} {period_date.date()}: Revenue=${period_data.get('revenue', 0):,.0f}")
                
            except Exception as e:
                logger.error(f"Error processing period data for {ticker}: {e}")
                continue
        
        if financial_records:
            await db.commit()
            # Refresh all records to get IDs
            for record in financial_records:
                if record.id is None:  # Only refresh new records
                    await db.refresh(record)
        
        logger.info(f"Successfully fetched {len(financial_records)} SEC financial records for {ticker}")
        return financial_records

    async def get_company_info(self, ticker: str) -> Dict[str, Any]:
        """Get company information from SEC"""
        try:
            cik = await self.get_company_cik(ticker)
            if not cik:
                return self._get_fallback_company_info(ticker)
            
            facts_data = await self.get_company_facts(cik)
            if not facts_data:
                return self._get_fallback_company_info(ticker)
            
            entity_info = facts_data.get('entityName', ticker)
            
            return {
                'name': entity_info,
                'cik': cik,
                'sector': 'Technology',  # SEC doesn't provide sector info directly
                'industry': 'Software',
                'business_description': f'{entity_info} - SEC registered company'
            }
            
        except Exception as e:
            logger.error(f"Error fetching SEC company info for {ticker}: {e}")
            return self._get_fallback_company_info(ticker)
    
    def _get_fallback_company_info(self, ticker: str) -> Dict[str, Any]:
        """Fallback company info"""
        return {
            'name': f'{ticker} Corporation',
            'cik': f'000{hash(ticker) % 1000000:06d}',
            'sector': 'Technology',
            'industry': 'Software',
            'business_description': f'{ticker} technology company'
        }