""" Real financial data service that combines price data with calculations """ from datetime import datetime, timezone, timedelta, date from typing import Dict, List, Optional, Tuple, Union import logging import asyncio import numpy as np from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy import select, and_, or_, desc from app.models.financial import Company, FinancialData, CalculatedMetrics, PriceData from app.schemas.financial import DataSource from app.services.price_data_service import PriceDataService from app.utils.date_utils import parse_period, quarters_to_date_range, resolve_time_parameters from app.core.config import settings logger = logging.getLogger(__name__) class FinancialService: """Real financial service that uses actual price data for calculations""" def __init__(self): self.price_service = PriceDataService() async def _get_ticker_max_range(self, ticker: str) -> Tuple[datetime, datetime]: """ Get maximum date range for a ticker by checking its listing date via yfinance_plus. Runs the blocking yfinance call in a thread pool with a 20s timeout. Returns: Tuple of (listing_date, current_date) or fallback to 20 years if yfinance unavailable """ try: import yfinance_plus as yf ticker_obj = yf.Ticker(ticker) loop = asyncio.get_event_loop() hist = await asyncio.wait_for( loop.run_in_executor( None, lambda: ticker_obj.history(period="max", interval="1mo") ), timeout=20, ) if not hist.empty: earliest_date = hist.index[0].to_pydatetime() if earliest_date.tzinfo is None: earliest_date = earliest_date.replace(tzinfo=timezone.utc) end_date = datetime.now(timezone.utc).replace(hour=23, minute=59, second=59, microsecond=0) sec_start = datetime(settings.SEC_DATA_START_YEAR, 1, 1, tzinfo=timezone.utc) actual_start = max(earliest_date, sec_start) logger.info(f"Found actual listing date for {ticker}: {actual_start.date()}") return actual_start, end_date except Exception as e: logger.warning(f"Could not get ticker info for {ticker}: {e}") # Fallback to 20-year max if yfinance_plus fails logger.info(f"Using fallback 20-year range for {ticker}") end_date = datetime.now(timezone.utc).replace(hour=23, minute=59, second=59, microsecond=0) start_date = end_date - timedelta(days=20 * 365.25) # 20 years sec_start = datetime(settings.SEC_DATA_START_YEAR, 1, 1, tzinfo=timezone.utc) actual_start = max(start_date, sec_start) return actual_start, end_date async def get_or_create_company_data( self, db: AsyncSession, ticker: str, start_date: Optional[Union[date, datetime]] = None, end_date: Optional[Union[date, datetime]] = None, quarters: Optional[List[str]] = None, period: Optional[str] = None, force_refresh: bool = False ) -> Dict: """ Get company data with real price-based calculations """ ticker = ticker.upper() # Resolve time parameters to standard datetime range. # _get_ticker_max_range is async, so handle "max" period before calling # the sync resolve_time_parameters helper. if period and period.lower() == "max": resolved_start, resolved_end = await self._get_ticker_max_range(ticker) else: resolved_start, resolved_end = resolve_time_parameters( start_date, end_date, quarters, period, ticker ) # Get or create company company = await self._get_or_create_company(db, ticker) # Get financial data from database or generate realistic data financial_data = await self._get_or_generate_financial_data( db, ticker, resolved_start, resolved_end, force_refresh ) # Get price data for calculations price_data = await self._get_price_data_for_period( db, ticker, resolved_start, resolved_end ) # Calculate metrics using real price data calculated_metrics = await self._calculate_real_metrics( db, ticker, financial_data, price_data, force_refresh ) return { "company": company, "financial_data": financial_data, "calculated_metrics": calculated_metrics } async def _get_or_create_company(self, db: AsyncSession, ticker: str) -> Company: """Get or create company record""" result = await db.execute( select(Company).where(Company.ticker == ticker) ) company = result.scalar_one_or_none() if not company: # Create company with basic info (in real implementation, this would fetch from SEC) company_info = self._get_default_company_info(ticker) company = Company( ticker=ticker, name=company_info["name"], cik=company_info["cik"], sector=company_info["sector"], industry=company_info["industry"], business_description=company_info["business_description"], created_at=datetime.now(timezone.utc), updated_at=datetime.now(timezone.utc) ) db.add(company) await db.commit() await db.refresh(company) return company def _get_default_company_info(self, ticker: str) -> Dict: """Get default company info (placeholder for real SEC data)""" company_defaults = { 'AAPL': { 'name': 'Apple Inc.', 'cik': '0000320193', 'sector': 'Technology', 'industry': 'Consumer Electronics', 'business_description': 'Technology company designing and manufacturing consumer electronics' }, 'MSFT': { 'name': 'Microsoft Corporation', 'cik': '0000789019', 'sector': 'Technology', 'industry': 'Software—Infrastructure', 'business_description': 'Software and cloud services company' }, 'TSLA': { 'name': 'Tesla Inc.', 'cik': '0001318605', 'sector': 'Consumer Cyclical', 'industry': 'Auto Manufacturers', 'business_description': 'Electric vehicle and clean energy company' }, 'NVDA': { 'name': 'NVIDIA Corporation', 'cik': '0001045810', 'sector': 'Technology', 'industry': 'Semiconductors', 'business_description': 'Semiconductor company specializing in graphics processing units' } } return company_defaults.get(ticker, { 'name': f'{ticker} Corporation', 'cik': f'000{hash(ticker) % 1000000:06d}', 'sector': 'Technology', 'industry': 'Software', 'business_description': f'{ticker} technology company' }) async def _get_or_generate_financial_data( self, db: AsyncSession, ticker: str, start_date: datetime, end_date: datetime, force_refresh: bool = False ) -> List[FinancialData]: """Get financial data from database or return empty list if no real data exists""" # First, try to get real data from SEC EDGAR if not force refresh if not force_refresh: try: from app.services.sec_edgar_service import SECEdgarService sec_service = SECEdgarService() real_financial_data = await sec_service.get_financial_data( db, ticker, start_date, end_date, force_refresh ) # If we got real data, return it if real_financial_data: logger.info(f"Found {len(real_financial_data)} real financial records for {ticker}") return real_financial_data else: logger.info(f"No real financial data found for {ticker}, returning empty list") return [] except Exception as e: logger.error(f"Error fetching real financial data for {ticker}: {e}") # Fall back to database check if SEC service fails # Check existing data in database (both real and estimated) result = await db.execute( select(FinancialData) .where( and_( FinancialData.ticker == ticker, FinancialData.period_date >= start_date, FinancialData.period_date <= end_date ) ) .order_by(FinancialData.period_date) ) existing_data = result.scalars().all() if existing_data and not force_refresh: return existing_data # If no real data and no existing data, return empty list instead of generating estimated data logger.info(f"No financial data available for {ticker} in the requested period") return [] async def _generate_realistic_financial_data( self, db: AsyncSession, ticker: str, start_date: datetime, end_date: datetime, force_refresh: bool = False ) -> List[FinancialData]: """Generate realistic financial data based on company size and sector""" # Get company base metrics from real market data if available base_metrics = await self._estimate_company_size(db, ticker) # Generate quarterly periods quarters = self._generate_quarterly_periods(start_date, end_date) financial_records = [] for i, quarter_end in enumerate(quarters): # Calculate growth factor based on time progression years_from_start = (quarter_end - start_date).days / 365.25 growth_factor = (1.05) ** years_from_start # 5% annual growth baseline # Add some realistic volatility volatility = np.random.normal(1.0, 0.08) # 8% volatility total_factor = growth_factor * volatility # Generate realistic financial metrics revenue = base_metrics['revenue'] * total_factor gross_profit = revenue * base_metrics['gross_margin'] operating_income = revenue * base_metrics['operating_margin'] net_income = revenue * base_metrics['net_margin'] # Balance sheet items total_assets = base_metrics['total_assets'] * total_factor total_equity = total_assets * base_metrics['equity_ratio'] total_debt = total_assets * base_metrics['debt_ratio'] cash = total_assets * base_metrics['cash_ratio'] shares_outstanding = base_metrics['shares_outstanding'] # Cash flow items operating_cash_flow = net_income * 1.15 # OCF typically higher than net income capex = revenue * 0.04 # 4% of revenue free_cash_flow = operating_cash_flow - capex eps = net_income / shares_outstanding if shares_outstanding > 0 else 0 # Check if record exists existing = await db.execute( select(FinancialData).where( and_( FinancialData.ticker == ticker, FinancialData.period_date == quarter_end, FinancialData.period_type == "quarterly" ) ) ) if existing.scalar_one_or_none() and not force_refresh: continue # Create or update financial data record financial_record = FinancialData( ticker=ticker, period_date=quarter_end, period_type="quarterly", filing_type="10-Q", revenue=revenue, gross_profit=gross_profit, operating_income=operating_income, net_income=net_income, eps=eps, total_assets=total_assets, total_equity=total_equity, total_debt=total_debt, cash=cash, shares_outstanding=shares_outstanding, operating_cash_flow=operating_cash_flow, free_cash_flow=free_cash_flow, capex=capex, data_source=DataSource.SEC_EDGAR.value, is_estimated=True, # Mark as estimated since we're generating it created_at=datetime.now(timezone.utc), updated_at=datetime.now(timezone.utc) ) # Delete existing if force refresh if force_refresh: await db.execute( select(FinancialData).where( and_( FinancialData.ticker == ticker, FinancialData.period_date == quarter_end, FinancialData.period_type == "quarterly" ) ) ) db.add(financial_record) financial_records.append(financial_record) await db.commit() # Refresh all records to get IDs for record in financial_records: await db.refresh(record) return financial_records async def _estimate_company_size(self, db: AsyncSession, ticker: str) -> Dict: """Estimate company size based on recent price data and industry""" # Get recent price data to estimate market cap recent_date = datetime.now(timezone.utc) - timedelta(days=30) result = await db.execute( select(PriceData) .where( and_( PriceData.ticker == ticker, PriceData.date >= recent_date ) ) .order_by(desc(PriceData.date)) .limit(1) ) recent_price = result.scalar_one_or_none() # Default metrics based on typical companies default_metrics = { 'revenue': 50_000_000_000, # $50B 'total_assets': 75_000_000_000, # $75B 'shares_outstanding': 1_000_000_000, # 1B shares 'gross_margin': 0.45, # 45% 'operating_margin': 0.15, # 15% 'net_margin': 0.12, # 12% 'equity_ratio': 0.40, # 40% 'debt_ratio': 0.25, # 25% 'cash_ratio': 0.10, # 10% } if recent_price: # Estimate company size based on current price estimated_market_cap = recent_price.close * default_metrics['shares_outstanding'] # Adjust metrics based on estimated market cap if estimated_market_cap > 1_000_000_000_000: # $1T+ (mega cap) scale_factor = 5.0 elif estimated_market_cap > 200_000_000_000: # $200B+ (large cap) scale_factor = 3.0 elif estimated_market_cap > 10_000_000_000: # $10B+ (mid cap) scale_factor = 1.5 else: # Small cap scale_factor = 0.5 default_metrics['revenue'] *= scale_factor default_metrics['total_assets'] *= scale_factor return default_metrics def _generate_quarterly_periods(self, start_date: datetime, end_date: datetime) -> List[datetime]: """Generate quarterly period end dates""" quarters = [] # Start from the first quarter end after start_date current_year = start_date.year quarter_ends = [ datetime(current_year, 3, 31, tzinfo=timezone.utc), datetime(current_year, 6, 30, tzinfo=timezone.utc), datetime(current_year, 9, 30, tzinfo=timezone.utc), datetime(current_year, 12, 31, tzinfo=timezone.utc), ] # Find first quarter end >= start_date for qe in quarter_ends: if qe >= start_date: quarters.append(qe) # Add subsequent years year = current_year + 1 while True: year_quarters = [ datetime(year, 3, 31, tzinfo=timezone.utc), datetime(year, 6, 30, tzinfo=timezone.utc), datetime(year, 9, 30, tzinfo=timezone.utc), datetime(year, 12, 31, tzinfo=timezone.utc), ] added_any = False for qe in year_quarters: if qe <= end_date: quarters.append(qe) added_any = True else: break if not added_any: break year += 1 return quarters async def _get_price_data_for_period( self, db: AsyncSession, ticker: str, start_date: datetime, end_date: datetime ) -> List[PriceData]: """Get price data for the specified period""" # Try to get from database first result = await db.execute( select(PriceData) .where( and_( PriceData.ticker == ticker, PriceData.date >= start_date, PriceData.date <= end_date ) ) .order_by(PriceData.date) ) price_data = result.scalars().all() # If no price data, try to fetch it if not price_data: try: price_data = await self.price_service.get_or_update_price_data( db, ticker, start_date, end_date, "1d", force_refresh=False ) except Exception as e: logger.warning(f"Could not fetch price data for {ticker}: {e}") price_data = [] return price_data async def _calculate_real_metrics( self, db: AsyncSession, ticker: str, financial_data: List[FinancialData], price_data: List[PriceData], force_refresh: bool = False ) -> List[CalculatedMetrics]: """Calculate metrics using real price data""" calculated_metrics = [] for financial_record in financial_data: period_date = financial_record.period_date # Check if metrics already exist existing_metrics = None if not force_refresh: existing = await db.execute( select(CalculatedMetrics).where( and_( CalculatedMetrics.ticker == ticker, CalculatedMetrics.period_date == period_date ) ) ) existing_metrics = existing.scalar_one_or_none() if existing_metrics: calculated_metrics.append(existing_metrics) continue # Find price data close to the period date price_at_period = self._find_price_near_date(price_data, period_date) if not price_at_period: logger.warning(f"No price data found for {ticker} near {period_date}") continue # Calculate valuation metrics using real price market_cap = price_at_period.close * financial_record.shares_outstanding if financial_record.shares_outstanding else None pe_ratio = None if financial_record.eps and financial_record.eps > 0: pe_ratio = price_at_period.close / financial_record.eps pb_ratio = None if financial_record.total_equity and financial_record.shares_outstanding: book_value_per_share = financial_record.total_equity / financial_record.shares_outstanding if book_value_per_share > 0: pb_ratio = price_at_period.close / book_value_per_share ps_ratio = None if financial_record.revenue and financial_record.shares_outstanding: revenue_per_share = financial_record.revenue / financial_record.shares_outstanding if revenue_per_share > 0: ps_ratio = price_at_period.close / revenue_per_share # Calculate profitability metrics roe = None if financial_record.net_income and financial_record.total_equity and financial_record.total_equity > 0: roe = financial_record.net_income / financial_record.total_equity roa = None if financial_record.net_income and financial_record.total_assets and financial_record.total_assets > 0: roa = financial_record.net_income / financial_record.total_assets gross_margin = None if financial_record.gross_profit and financial_record.revenue and financial_record.revenue > 0: gross_margin = financial_record.gross_profit / financial_record.revenue operating_margin = None if financial_record.operating_income and financial_record.revenue and financial_record.revenue > 0: operating_margin = financial_record.operating_income / financial_record.revenue net_margin = None if financial_record.net_income and financial_record.revenue and financial_record.revenue > 0: net_margin = financial_record.net_income / financial_record.revenue # Calculate debt ratios debt_to_equity = None if financial_record.total_debt and financial_record.total_equity and financial_record.total_equity > 0: debt_to_equity = financial_record.total_debt / financial_record.total_equity debt_to_assets = None if financial_record.total_debt and financial_record.total_assets and financial_record.total_assets > 0: debt_to_assets = financial_record.total_debt / financial_record.total_assets # Calculate cash flow metrics ocf_margin = None if financial_record.operating_cash_flow and financial_record.revenue and financial_record.revenue > 0: ocf_margin = financial_record.operating_cash_flow / financial_record.revenue fcf_margin = None if financial_record.free_cash_flow and financial_record.revenue and financial_record.revenue > 0: fcf_margin = financial_record.free_cash_flow / financial_record.revenue # Create calculated metrics record metrics = CalculatedMetrics( ticker=ticker, calculation_date=datetime.now(timezone.utc), period_date=period_date, pe_ratio=pe_ratio, pb_ratio=pb_ratio, ps_ratio=ps_ratio, roe=roe, roa=roa, gross_margin=gross_margin, operating_margin=operating_margin, net_margin=net_margin, debt_to_equity=debt_to_equity, debt_to_assets=debt_to_assets, ocf_margin=ocf_margin, fcf_margin=fcf_margin, market_cap=market_cap, created_at=datetime.now(timezone.utc), updated_at=datetime.now(timezone.utc) ) db.add(metrics) calculated_metrics.append(metrics) if calculated_metrics: await db.commit() for metrics in calculated_metrics: await db.refresh(metrics) return calculated_metrics def _find_price_near_date(self, price_data: List[PriceData], target_date: datetime) -> Optional[PriceData]: """Find price data closest to the target date""" if not price_data: return None # Convert target_date to date for comparison target_date_only = target_date.date() closest_price = None min_diff = float('inf') for price in price_data: price_date = price.date.date() if hasattr(price.date, 'date') else price.date diff = abs((price_date - target_date_only).days) if diff < min_diff: min_diff = diff closest_price = price return closest_price