""" Real financial data service that combines price data with calculations """ from datetime import datetime, timezone, timedelta, date from typing import Dict, List, Optional, Tuple, Union import logging import asyncio import numpy as np from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy import select, and_, or_, desc from app.models.financial import Company, FinancialData, CalculatedMetrics, PriceData from app.schemas.financial import DataSource from app.services.price_data_service import PriceDataService from app.utils.date_utils import parse_period, quarters_to_date_range, resolve_time_parameters from app.core.config import settings logger = logging.getLogger(__name__) class FinancialService: """Real financial service that uses actual price data for calculations""" def __init__(self): self.price_service = PriceDataService() async def _get_ticker_max_range(self, ticker: str) -> Tuple[datetime, datetime]: """ Get maximum date range for a ticker by checking its listing date via yfinance_plus. Runs the blocking yfinance call in a thread pool with a 20s timeout. Returns: Tuple of (listing_date, current_date) or fallback to 20 years if yfinance unavailable """ try: import yfinance_plus as yf ticker_obj = yf.Ticker(ticker) loop = asyncio.get_event_loop() hist = await asyncio.wait_for( loop.run_in_executor( None, lambda: ticker_obj.history(period="max", interval="1mo") ), timeout=20, ) if not hist.empty: earliest_date = hist.index[0].to_pydatetime() if earliest_date.tzinfo is None: earliest_date = earliest_date.replace(tzinfo=timezone.utc) end_date = datetime.now(timezone.utc).replace(hour=23, minute=59, second=59, microsecond=0) sec_start = datetime(settings.SEC_DATA_START_YEAR, 1, 1, tzinfo=timezone.utc) actual_start = max(earliest_date, sec_start) logger.info(f"Found actual listing date for {ticker}: {actual_start.date()}") return actual_start, end_date except Exception as e: logger.warning(f"Could not get ticker info for {ticker}: {e}") # Fallback to 20-year max if yfinance_plus fails logger.info(f"Using fallback 20-year range for {ticker}") end_date = datetime.now(timezone.utc).replace(hour=23, minute=59, second=59, microsecond=0) start_date = end_date - timedelta(days=20 * 365.25) # 20 years sec_start = datetime(settings.SEC_DATA_START_YEAR, 1, 1, tzinfo=timezone.utc) actual_start = max(start_date, sec_start) return actual_start, end_date async def get_or_create_company_data( self, db: AsyncSession, ticker: str, start_date: Optional[Union[date, datetime]] = None, end_date: Optional[Union[date, datetime]] = None, quarters: Optional[List[str]] = None, period: Optional[str] = None, force_refresh: bool = False ) -> Dict: """ Get company data with real price-based calculations """ ticker = ticker.upper() # Resolve time parameters to standard datetime range. # _get_ticker_max_range is async, so handle "max" period before calling # the sync resolve_time_parameters helper. if period and period.lower() == "max": resolved_start, resolved_end = await self._get_ticker_max_range(ticker) else: resolved_start, resolved_end = resolve_time_parameters( start_date, end_date, quarters, period, ticker ) # Get or create company — always succeeds (may have NULL sector for truly unknown tickers) company = await self._get_or_create_company(db, ticker) # Fetch financials and price data independently; failures return empty lists. try: financial_data = await self._get_or_generate_financial_data( db, ticker, resolved_start, resolved_end, force_refresh ) except Exception as e: logger.warning("Financial data fetch failed for %s: %s", ticker, e) financial_data = [] try: price_data = await self._get_price_data_for_period( db, ticker, resolved_start, resolved_end ) except Exception as e: logger.warning("Price data fetch failed for %s: %s", ticker, e) price_data = [] # Calculate metrics using real price data calculated_metrics = await self._calculate_real_metrics( db, ticker, financial_data, price_data, force_refresh ) return { "company": company, "financial_data": financial_data, "calculated_metrics": calculated_metrics } @staticmethod def _is_placeholder(company: Company) -> bool: """True when the row was created by the old hardcoded-defaults path.""" return bool( company.sector == "Technology" and company.industry == "Software" and company.name and company.name.endswith(" Corporation") ) async def _get_or_create_company(self, db: AsyncSession, ticker: str) -> Company: """Get or create company record, enriching via CompanyMetadataService.""" result = await db.execute(select(Company).where(Company.ticker == ticker)) company = result.scalar_one_or_none() # Fast path: row exists with real sector (not a hardcoded placeholder) if company and company.sector and not self._is_placeholder(company): return company # Enrich via registry + yfinance try: from app.services import company_metadata_service as cms meta = await cms.get_metadata(db, ticker) except ValueError: # Invalid ticker — still create a minimal placeholder so the rest of the # financial pipeline doesn't break. meta = { "name": f"{ticker} Corporation", "cik": None, "exchange": None, "sector": None, "industry": None, "country": None, "market_cap": None, "business_description": None, } except Exception as e: logger.warning("CompanyMetadataService failed for %s: %s — using placeholder", ticker, e) meta = { "name": f"{ticker} Corporation", "cik": None, "exchange": None, "sector": None, "industry": None, "country": None, "market_cap": None, "business_description": None, } # cms.get_metadata already upserts the Company row; re-fetch or create if missing. result = await db.execute(select(Company).where(Company.ticker == ticker)) company = result.scalar_one_or_none() if not company: now = datetime.now(timezone.utc) company = Company( ticker=ticker, name=meta.get("name") or f"{ticker} Corporation", cik=meta.get("cik"), exchange=meta.get("exchange"), sector=meta.get("sector"), industry=meta.get("industry"), country=meta.get("country"), market_cap=meta.get("market_cap"), business_description=meta.get("business_description"), created_at=now, updated_at=now, ) db.add(company) await db.commit() await db.refresh(company) return company async def _get_or_generate_financial_data( self, db: AsyncSession, ticker: str, start_date: datetime, end_date: datetime, force_refresh: bool = False ) -> List[FinancialData]: """Get financial data from database or return empty list if no real data exists""" # First, try to get real data from SEC EDGAR if not force refresh if not force_refresh: try: from app.services.sec_edgar_service import SECEdgarService sec_service = SECEdgarService() real_financial_data = await sec_service.get_financial_data( db, ticker, start_date, end_date, force_refresh ) # If we got real data, return it if real_financial_data: logger.info(f"Found {len(real_financial_data)} real financial records for {ticker}") return real_financial_data else: logger.info(f"No real financial data found for {ticker}, returning empty list") return [] except Exception as e: logger.error(f"Error fetching real financial data for {ticker}: {e}") # Fall back to database check if SEC service fails # Check existing data in database (both real and estimated) result = await db.execute( select(FinancialData) .where( and_( FinancialData.ticker == ticker, FinancialData.period_date >= start_date, FinancialData.period_date <= end_date ) ) .order_by(FinancialData.period_date) ) existing_data = result.scalars().all() if existing_data and not force_refresh: return existing_data # If no real data and no existing data, return empty list instead of generating estimated data logger.info(f"No financial data available for {ticker} in the requested period") return [] async def _generate_realistic_financial_data( self, db: AsyncSession, ticker: str, start_date: datetime, end_date: datetime, force_refresh: bool = False ) -> List[FinancialData]: """Generate realistic financial data based on company size and sector""" # Get company base metrics from real market data if available base_metrics = await self._estimate_company_size(db, ticker) # Generate quarterly periods quarters = self._generate_quarterly_periods(start_date, end_date) financial_records = [] for i, quarter_end in enumerate(quarters): # Calculate growth factor based on time progression years_from_start = (quarter_end - start_date).days / 365.25 growth_factor = (1.05) ** years_from_start # 5% annual growth baseline # Add some realistic volatility volatility = np.random.normal(1.0, 0.08) # 8% volatility total_factor = growth_factor * volatility # Generate realistic financial metrics revenue = base_metrics['revenue'] * total_factor gross_profit = revenue * base_metrics['gross_margin'] operating_income = revenue * base_metrics['operating_margin'] net_income = revenue * base_metrics['net_margin'] # Balance sheet items total_assets = base_metrics['total_assets'] * total_factor total_equity = total_assets * base_metrics['equity_ratio'] total_debt = total_assets * base_metrics['debt_ratio'] cash = total_assets * base_metrics['cash_ratio'] shares_outstanding = base_metrics['shares_outstanding'] # Cash flow items operating_cash_flow = net_income * 1.15 # OCF typically higher than net income capex = revenue * 0.04 # 4% of revenue free_cash_flow = operating_cash_flow - capex eps = net_income / shares_outstanding if shares_outstanding > 0 else 0 # Check if record exists existing = await db.execute( select(FinancialData).where( and_( FinancialData.ticker == ticker, FinancialData.period_date == quarter_end, FinancialData.period_type == "quarterly" ) ) ) if existing.scalar_one_or_none() and not force_refresh: continue # Create or update financial data record financial_record = FinancialData( ticker=ticker, period_date=quarter_end, period_type="quarterly", filing_type="10-Q", revenue=revenue, gross_profit=gross_profit, operating_income=operating_income, net_income=net_income, eps=eps, total_assets=total_assets, total_equity=total_equity, total_debt=total_debt, cash=cash, shares_outstanding=shares_outstanding, operating_cash_flow=operating_cash_flow, free_cash_flow=free_cash_flow, capex=capex, data_source=DataSource.SEC_EDGAR.value, is_estimated=True, # Mark as estimated since we're generating it created_at=datetime.now(timezone.utc), updated_at=datetime.now(timezone.utc) ) # Delete existing if force refresh if force_refresh: await db.execute( select(FinancialData).where( and_( FinancialData.ticker == ticker, FinancialData.period_date == quarter_end, FinancialData.period_type == "quarterly" ) ) ) db.add(financial_record) financial_records.append(financial_record) await db.commit() # Refresh all records to get IDs for record in financial_records: await db.refresh(record) return financial_records async def _estimate_company_size(self, db: AsyncSession, ticker: str) -> Dict: """Estimate company size based on recent price data and industry""" # Get recent price data to estimate market cap recent_date = datetime.now(timezone.utc) - timedelta(days=30) result = await db.execute( select(PriceData) .where( and_( PriceData.ticker == ticker, PriceData.date >= recent_date ) ) .order_by(desc(PriceData.date)) .limit(1) ) recent_price = result.scalar_one_or_none() # Default metrics based on typical companies default_metrics = { 'revenue': 50_000_000_000, # $50B 'total_assets': 75_000_000_000, # $75B 'shares_outstanding': 1_000_000_000, # 1B shares 'gross_margin': 0.45, # 45% 'operating_margin': 0.15, # 15% 'net_margin': 0.12, # 12% 'equity_ratio': 0.40, # 40% 'debt_ratio': 0.25, # 25% 'cash_ratio': 0.10, # 10% } if recent_price: # Estimate company size based on current price estimated_market_cap = recent_price.close * default_metrics['shares_outstanding'] # Adjust metrics based on estimated market cap if estimated_market_cap > 1_000_000_000_000: # $1T+ (mega cap) scale_factor = 5.0 elif estimated_market_cap > 200_000_000_000: # $200B+ (large cap) scale_factor = 3.0 elif estimated_market_cap > 10_000_000_000: # $10B+ (mid cap) scale_factor = 1.5 else: # Small cap scale_factor = 0.5 default_metrics['revenue'] *= scale_factor default_metrics['total_assets'] *= scale_factor return default_metrics def _generate_quarterly_periods(self, start_date: datetime, end_date: datetime) -> List[datetime]: """Generate quarterly period end dates""" quarters = [] # Start from the first quarter end after start_date current_year = start_date.year quarter_ends = [ datetime(current_year, 3, 31, tzinfo=timezone.utc), datetime(current_year, 6, 30, tzinfo=timezone.utc), datetime(current_year, 9, 30, tzinfo=timezone.utc), datetime(current_year, 12, 31, tzinfo=timezone.utc), ] # Find first quarter end >= start_date for qe in quarter_ends: if qe >= start_date: quarters.append(qe) # Add subsequent years year = current_year + 1 while True: year_quarters = [ datetime(year, 3, 31, tzinfo=timezone.utc), datetime(year, 6, 30, tzinfo=timezone.utc), datetime(year, 9, 30, tzinfo=timezone.utc), datetime(year, 12, 31, tzinfo=timezone.utc), ] added_any = False for qe in year_quarters: if qe <= end_date: quarters.append(qe) added_any = True else: break if not added_any: break year += 1 return quarters async def _get_price_data_for_period( self, db: AsyncSession, ticker: str, start_date: datetime, end_date: datetime ) -> List[PriceData]: """Get price data for the specified period""" # Try to get from database first result = await db.execute( select(PriceData) .where( and_( PriceData.ticker == ticker, PriceData.date >= start_date, PriceData.date <= end_date ) ) .order_by(PriceData.date) ) price_data = result.scalars().all() # If no price data, try to fetch it if not price_data: try: price_data = await self.price_service.get_or_update_price_data( ticker, start_date, end_date, "1d", force_refresh=False ) except Exception as e: logger.warning(f"Could not fetch price data for {ticker}: {e}") price_data = [] return price_data async def _calculate_real_metrics( self, db: AsyncSession, ticker: str, financial_data: List[FinancialData], price_data: List[PriceData], force_refresh: bool = False ) -> List[CalculatedMetrics]: """Calculate metrics using real price data""" calculated_metrics = [] for financial_record in financial_data: period_date = financial_record.period_date # Check if metrics already exist existing_metrics = None if not force_refresh: existing = await db.execute( select(CalculatedMetrics).where( and_( CalculatedMetrics.ticker == ticker, CalculatedMetrics.period_date == period_date ) ) ) existing_metrics = existing.scalar_one_or_none() if existing_metrics: calculated_metrics.append(existing_metrics) continue # Find price data close to the period date price_at_period = self._find_price_near_date(price_data, period_date) if not price_at_period: logger.warning(f"No price data found for {ticker} near {period_date}") continue # Calculate valuation metrics using real price market_cap = price_at_period.close * financial_record.shares_outstanding if financial_record.shares_outstanding else None pe_ratio = None if financial_record.eps and financial_record.eps > 0: pe_ratio = price_at_period.close / financial_record.eps pb_ratio = None if financial_record.total_equity and financial_record.shares_outstanding: book_value_per_share = financial_record.total_equity / financial_record.shares_outstanding if book_value_per_share > 0: pb_ratio = price_at_period.close / book_value_per_share ps_ratio = None if financial_record.revenue and financial_record.shares_outstanding: revenue_per_share = financial_record.revenue / financial_record.shares_outstanding if revenue_per_share > 0: ps_ratio = price_at_period.close / revenue_per_share # Calculate profitability metrics roe = None if financial_record.net_income and financial_record.total_equity and financial_record.total_equity > 0: roe = financial_record.net_income / financial_record.total_equity roa = None if financial_record.net_income and financial_record.total_assets and financial_record.total_assets > 0: roa = financial_record.net_income / financial_record.total_assets gross_margin = None if financial_record.gross_profit and financial_record.revenue and financial_record.revenue > 0: gross_margin = financial_record.gross_profit / financial_record.revenue operating_margin = None if financial_record.operating_income and financial_record.revenue and financial_record.revenue > 0: operating_margin = financial_record.operating_income / financial_record.revenue net_margin = None if financial_record.net_income and financial_record.revenue and financial_record.revenue > 0: net_margin = financial_record.net_income / financial_record.revenue # Calculate debt ratios debt_to_equity = None if financial_record.total_debt and financial_record.total_equity and financial_record.total_equity > 0: debt_to_equity = financial_record.total_debt / financial_record.total_equity debt_to_assets = None if financial_record.total_debt and financial_record.total_assets and financial_record.total_assets > 0: debt_to_assets = financial_record.total_debt / financial_record.total_assets # Calculate cash flow metrics ocf_margin = None if financial_record.operating_cash_flow and financial_record.revenue and financial_record.revenue > 0: ocf_margin = financial_record.operating_cash_flow / financial_record.revenue fcf_margin = None if financial_record.free_cash_flow and financial_record.revenue and financial_record.revenue > 0: fcf_margin = financial_record.free_cash_flow / financial_record.revenue # Create calculated metrics record metrics = CalculatedMetrics( ticker=ticker, calculation_date=datetime.now(timezone.utc), period_date=period_date, pe_ratio=pe_ratio, pb_ratio=pb_ratio, ps_ratio=ps_ratio, roe=roe, roa=roa, gross_margin=gross_margin, operating_margin=operating_margin, net_margin=net_margin, debt_to_equity=debt_to_equity, debt_to_assets=debt_to_assets, ocf_margin=ocf_margin, fcf_margin=fcf_margin, market_cap=market_cap, created_at=datetime.now(timezone.utc), updated_at=datetime.now(timezone.utc) ) db.add(metrics) calculated_metrics.append(metrics) if calculated_metrics: await db.commit() for metrics in calculated_metrics: await db.refresh(metrics) return calculated_metrics def _find_price_near_date(self, price_data: List[PriceData], target_date: datetime) -> Optional[PriceData]: """Find price data closest to the target date""" if not price_data: return None # Convert target_date to date for comparison target_date_only = target_date.date() closest_price = None min_diff = float('inf') for price in price_data: price_date = price.date.date() if hasattr(price.date, 'date') else price.date diff = abs((price_date - target_date_only).days) if diff < min_diff: min_diff = diff closest_price = price return closest_price