stock-oracle/app/services/real_sec_financial_service.py

"""
Real SEC Financial Data Service
Uses direct SEC EDGAR API to fetch actual SEC filing data and yfinance-plus for price data only
"""

from datetime import datetime, timezone, timedelta
from typing import Dict, List, Optional, Tuple
import logging
import numpy as np
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, and_, or_, desc

from app.models.financial import Company, FinancialData, CalculatedMetrics, PriceData
from app.schemas.financial import DataSource
from app.services.price_data_service import PriceDataService
from app.services.sec_edgar_service import SECEdgarService
from app.core.config import settings

# Import yfinance-plus only for price data
try:
    import yfinance_plus as yf
    YFINANCE_AVAILABLE = True
    logger = logging.getLogger(__name__)
    logger.info("yfinance-plus imported for price data only")
except ImportError:
    logger = logging.getLogger(__name__)
    logger.error("yfinance-plus not available for price data")
    YFINANCE_AVAILABLE = False

logger = logging.getLogger(__name__)

class RealSECFinancialService:
    """Real financial service that uses actual SEC EDGAR API"""
    
    def __init__(self):
        self.price_service = PriceDataService()
        self.sec_service = SECEdgarService()
        logger.info("SEC EDGAR service initialized")
    
    async def get_or_create_company_data(
        self,
        db: AsyncSession,
        ticker: str,
        start_date: datetime,
        end_date: datetime,
        force_refresh: bool = False
    ) -> Dict:
        """
        Get company data with real SEC financial data
        """
        ticker = ticker.upper()
        
        # Get or create company
        company = await self._get_or_create_company(db, ticker)
        
        # Get real financial data from SEC EDGAR API
        financial_data = await self.sec_service.get_financial_data(
            db, ticker, start_date, end_date, force_refresh
        )
        
        # Get price data for calculations
        price_data = await self._get_price_data_for_period(
            db, ticker, start_date, end_date
        )
        
        # Calculate metrics using real data
        calculated_metrics = await self._calculate_real_metrics(
            db, ticker, financial_data, price_data, force_refresh
        )
        
        return {
            "company": company,
            "financial_data": financial_data,
            "calculated_metrics": calculated_metrics
        }
    
    async def _get_or_create_company(self, db: AsyncSession, ticker: str) -> Company:
        """Get or create company record using real SEC data"""
        result = await db.execute(
            select(Company).where(Company.ticker == ticker)
        )
        company = result.scalar_one_or_none()
        
        if not company:
            # Get real company info from SEC
            company_info = await self._fetch_company_info_from_sec(ticker)
            company = Company(
                ticker=ticker,
                name=company_info["name"],
                cik=company_info["cik"],
                sector=company_info["sector"],
                industry=company_info["industry"],
                business_description=company_info["business_description"],
                created_at=datetime.now(timezone.utc),
                updated_at=datetime.now(timezone.utc)
            )
            db.add(company)
            await db.commit()
            await db.refresh(company)
        
        return company
    
    async def _fetch_company_info_from_sec(self, ticker: str) -> Dict:
        """Fetch real company information from SEC EDGAR API"""
        try:
            return await self.sec_service.get_company_info(ticker)
        except Exception as e:
            logger.error(f"Error fetching SEC company info for {ticker}: {e}")
            return self._get_fallback_company_info(ticker)
    
    def _get_fallback_company_info(self, ticker: str) -> Dict:
        """Fallback company info if SEC is not available"""
        company_defaults = {
            'AAPL': {
                'name': 'Apple Inc.',
                'cik': '0000320193',
                'sector': 'Technology',
                'industry': 'Consumer Electronics',
                'business_description': 'Technology company designing and manufacturing consumer electronics'
            },
            'MSFT': {
                'name': 'Microsoft Corporation',
                'cik': '0000789019',
                'sector': 'Technology',
                'industry': 'Software—Infrastructure',
                'business_description': 'Software and cloud services company'
            },
            'TSLA': {
                'name': 'Tesla Inc.',
                'cik': '0001318605',
                'sector': 'Consumer Cyclical',
                'industry': 'Auto Manufacturers',
                'business_description': 'Electric vehicle and clean energy company'
            },
            'NVDA': {
                'name': 'NVIDIA Corporation',
                'cik': '0001045810',
                'sector': 'Technology',
                'industry': 'Semiconductors',
                'business_description': 'Semiconductor company specializing in graphics processing units'
            }
        }
        
        return company_defaults.get(ticker, {
            'name': f'{ticker} Corporation',
            'cik': f'000{hash(ticker) % 1000000:06d}',
            'sector': 'Technology',
            'industry': 'Software',
            'business_description': f'{ticker} technology company'
        })
    
    # SEC financial data fetching is now handled by SECEdgarService
    
    # Financial data fetching is now handled by SECEdgarService only
    # yfinance is only used for price data via PriceDataService
    
    # All financial data processing is now handled by SECEdgarService
    
    async def _get_price_data_for_period(
        self,
        db: AsyncSession,
        ticker: str,
        start_date: datetime,
        end_date: datetime
    ) -> List[PriceData]:
        """Get price data for the specified period.

        Failures (e.g. unknown ticker, yfinance error) are swallowed so that a
        price-fetch problem never causes the financial endpoint to return 500.
        """
        try:
            return await self.price_service.get_or_update_price_data(
                db, ticker, start_date, end_date, "1d", force_refresh=False
            )
        except Exception as e:
            logger.warning(
                f"Could not fetch price data for {ticker} during financial data processing: {e}"
            )
            return []
    
    async def _calculate_real_metrics(
        self,
        db: AsyncSession,
        ticker: str,
        financial_data: List[FinancialData],
        price_data: List[PriceData],
        force_refresh: bool = False
    ) -> List[CalculatedMetrics]:
        """Calculate metrics using real financial and price data"""
        
        calculated_metrics = []
        
        for financial_record in financial_data:
            period_date = financial_record.period_date
            
            # Check if metrics already exist
            existing_metrics = None
            if not force_refresh:
                existing = await db.execute(
                    select(CalculatedMetrics).where(
                        and_(
                            CalculatedMetrics.ticker == ticker,
                            CalculatedMetrics.period_date == period_date
                        )
                    ).limit(1)
                )
                existing_metrics = existing.scalar_one_or_none()
                if existing_metrics:
                    calculated_metrics.append(existing_metrics)
                    continue
            
            # Find price data close to the period date
            price_at_period = self._find_price_near_date(price_data, period_date)
            
            if not price_at_period:
                logger.warning(f"No price data found for {ticker} near {period_date}")
                continue
            
            # Calculate valuation metrics using real price and real financial data
            market_cap = None
            if price_at_period.close and financial_record.shares_outstanding:
                market_cap = price_at_period.close * financial_record.shares_outstanding
            
            pe_ratio = None
            if financial_record.eps and financial_record.eps > 0 and price_at_period.close:
                pe_ratio = price_at_period.close / financial_record.eps
            
            pb_ratio = None
            if (financial_record.total_equity and financial_record.shares_outstanding and 
                financial_record.shares_outstanding > 0 and price_at_period.close):
                book_value_per_share = financial_record.total_equity / financial_record.shares_outstanding
                if book_value_per_share > 0:
                    pb_ratio = price_at_period.close / book_value_per_share
            
            ps_ratio = None
            if (financial_record.revenue and financial_record.shares_outstanding and 
                financial_record.shares_outstanding > 0 and price_at_period.close):
                revenue_per_share = financial_record.revenue / financial_record.shares_outstanding
                if revenue_per_share > 0:
                    ps_ratio = price_at_period.close / revenue_per_share
            
            # Calculate profitability metrics
            roe = None
            if (financial_record.net_income and financial_record.total_equity and 
                financial_record.total_equity > 0):
                roe = financial_record.net_income / financial_record.total_equity
            
            roa = None
            if (financial_record.net_income and financial_record.total_assets and 
                financial_record.total_assets > 0):
                roa = financial_record.net_income / financial_record.total_assets
            
            gross_margin = None
            if (financial_record.gross_profit and financial_record.revenue and 
                financial_record.revenue > 0):
                gross_margin = financial_record.gross_profit / financial_record.revenue
            
            operating_margin = None
            if (financial_record.operating_income and financial_record.revenue and 
                financial_record.revenue > 0):
                operating_margin = financial_record.operating_income / financial_record.revenue
            
            net_margin = None
            if (financial_record.net_income and financial_record.revenue and 
                financial_record.revenue > 0):
                net_margin = financial_record.net_income / financial_record.revenue
            
            # Calculate debt ratios
            debt_to_equity = None
            if (financial_record.total_debt and financial_record.total_equity and 
                financial_record.total_equity > 0):
                debt_to_equity = financial_record.total_debt / financial_record.total_equity
            
            debt_to_assets = None
            if (financial_record.total_debt and financial_record.total_assets and 
                financial_record.total_assets > 0):
                debt_to_assets = financial_record.total_debt / financial_record.total_assets
            
            # Calculate cash flow metrics
            ocf_margin = None
            if (financial_record.operating_cash_flow and financial_record.revenue and 
                financial_record.revenue > 0):
                ocf_margin = financial_record.operating_cash_flow / financial_record.revenue
            
            fcf_margin = None
            if (financial_record.free_cash_flow and financial_record.revenue and 
                financial_record.revenue > 0):
                fcf_margin = financial_record.free_cash_flow / financial_record.revenue
            
            # Create calculated metrics record
            metrics = CalculatedMetrics(
                ticker=ticker,
                calculation_date=datetime.now(timezone.utc),
                period_date=period_date,
                pe_ratio=pe_ratio,
                pb_ratio=pb_ratio,
                ps_ratio=ps_ratio,
                roe=roe,
                roa=roa,
                gross_margin=gross_margin,
                operating_margin=operating_margin,
                net_margin=net_margin,
                debt_to_equity=debt_to_equity,
                debt_to_assets=debt_to_assets,
                ocf_margin=ocf_margin,
                fcf_margin=fcf_margin,
                market_cap=market_cap,
                created_at=datetime.now(timezone.utc),
                updated_at=datetime.now(timezone.utc)
            )
            
            db.add(metrics)
            calculated_metrics.append(metrics)
        
        if calculated_metrics:
            await db.commit()
            for metrics in calculated_metrics:
                await db.refresh(metrics)
        
        return calculated_metrics
    
    def _find_price_near_date(self, price_data: List[PriceData], target_date: datetime) -> Optional[PriceData]:
        """Find price data closest to the target date"""
        if not price_data:
            return None
        
        # Convert target_date to date for comparison
        target_date_only = target_date.date()
        
        closest_price = None
        min_diff = float('inf')
        
        for price in price_data:
            price_date = price.date.date() if hasattr(price.date, 'date') else price.date
            diff = abs((price_date - target_date_only).days)
            
            if diff < min_diff:
                min_diff = diff
                closest_price = price
        
        return closest_price


# Import pandas for data processing
try:
    import pandas as pd
except ImportError:
    logger.error("pandas not available - real SEC financial service will not work")
    pd = None