You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
627 lines
25 KiB
Python
627 lines
25 KiB
Python
"""
|
|
Real financial data service that combines price data with calculations
|
|
"""
|
|
|
|
from datetime import datetime, timezone, timedelta, date
|
|
from typing import Dict, List, Optional, Tuple, Union
|
|
import logging
|
|
import asyncio
|
|
import numpy as np
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
from sqlalchemy import select, and_, or_, desc
|
|
|
|
from app.models.financial import Company, FinancialData, CalculatedMetrics, PriceData
|
|
from app.schemas.financial import DataSource
|
|
from app.services.price_data_service import PriceDataService
|
|
from app.utils.date_utils import parse_period, quarters_to_date_range, resolve_time_parameters
|
|
from app.core.config import settings
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
class FinancialService:
|
|
"""Real financial service that uses actual price data for calculations"""
|
|
|
|
def __init__(self):
|
|
self.price_service = PriceDataService()
|
|
|
|
async def _get_ticker_max_range(self, ticker: str) -> Tuple[datetime, datetime]:
|
|
"""
|
|
Get maximum date range for a ticker by checking its listing date via yfinance_plus.
|
|
Runs the blocking yfinance call in a thread pool with a 20s timeout.
|
|
|
|
Returns:
|
|
Tuple of (listing_date, current_date) or fallback to 20 years if yfinance unavailable
|
|
"""
|
|
try:
|
|
import yfinance_plus as yf
|
|
|
|
ticker_obj = yf.Ticker(ticker)
|
|
loop = asyncio.get_event_loop()
|
|
hist = await asyncio.wait_for(
|
|
loop.run_in_executor(
|
|
None,
|
|
lambda: ticker_obj.history(period="max", interval="1mo")
|
|
),
|
|
timeout=20,
|
|
)
|
|
|
|
if not hist.empty:
|
|
earliest_date = hist.index[0].to_pydatetime()
|
|
if earliest_date.tzinfo is None:
|
|
earliest_date = earliest_date.replace(tzinfo=timezone.utc)
|
|
|
|
end_date = datetime.now(timezone.utc).replace(hour=23, minute=59, second=59, microsecond=0)
|
|
sec_start = datetime(settings.SEC_DATA_START_YEAR, 1, 1, tzinfo=timezone.utc)
|
|
actual_start = max(earliest_date, sec_start)
|
|
|
|
logger.info(f"Found actual listing date for {ticker}: {actual_start.date()}")
|
|
return actual_start, end_date
|
|
|
|
except Exception as e:
|
|
logger.warning(f"Could not get ticker info for {ticker}: {e}")
|
|
|
|
# Fallback to 20-year max if yfinance_plus fails
|
|
logger.info(f"Using fallback 20-year range for {ticker}")
|
|
end_date = datetime.now(timezone.utc).replace(hour=23, minute=59, second=59, microsecond=0)
|
|
start_date = end_date - timedelta(days=20 * 365.25) # 20 years
|
|
|
|
sec_start = datetime(settings.SEC_DATA_START_YEAR, 1, 1, tzinfo=timezone.utc)
|
|
actual_start = max(start_date, sec_start)
|
|
|
|
return actual_start, end_date
|
|
|
|
async def get_or_create_company_data(
|
|
self,
|
|
db: AsyncSession,
|
|
ticker: str,
|
|
start_date: Optional[Union[date, datetime]] = None,
|
|
end_date: Optional[Union[date, datetime]] = None,
|
|
quarters: Optional[List[str]] = None,
|
|
period: Optional[str] = None,
|
|
force_refresh: bool = False
|
|
) -> Dict:
|
|
"""
|
|
Get company data with real price-based calculations
|
|
"""
|
|
ticker = ticker.upper()
|
|
|
|
# Resolve time parameters to standard datetime range.
|
|
# _get_ticker_max_range is async, so handle "max" period before calling
|
|
# the sync resolve_time_parameters helper.
|
|
if period and period.lower() == "max":
|
|
resolved_start, resolved_end = await self._get_ticker_max_range(ticker)
|
|
else:
|
|
resolved_start, resolved_end = resolve_time_parameters(
|
|
start_date, end_date, quarters, period, ticker
|
|
)
|
|
|
|
# Get or create company
|
|
company = await self._get_or_create_company(db, ticker)
|
|
|
|
# Get financial data from database or generate realistic data
|
|
financial_data = await self._get_or_generate_financial_data(
|
|
db, ticker, resolved_start, resolved_end, force_refresh
|
|
)
|
|
|
|
# Get price data for calculations
|
|
price_data = await self._get_price_data_for_period(
|
|
db, ticker, resolved_start, resolved_end
|
|
)
|
|
|
|
# Calculate metrics using real price data
|
|
calculated_metrics = await self._calculate_real_metrics(
|
|
db, ticker, financial_data, price_data, force_refresh
|
|
)
|
|
|
|
return {
|
|
"company": company,
|
|
"financial_data": financial_data,
|
|
"calculated_metrics": calculated_metrics
|
|
}
|
|
|
|
async def _get_or_create_company(self, db: AsyncSession, ticker: str) -> Company:
|
|
"""Get or create company record"""
|
|
result = await db.execute(
|
|
select(Company).where(Company.ticker == ticker)
|
|
)
|
|
company = result.scalar_one_or_none()
|
|
|
|
if not company:
|
|
# Create company with basic info (in real implementation, this would fetch from SEC)
|
|
company_info = self._get_default_company_info(ticker)
|
|
company = Company(
|
|
ticker=ticker,
|
|
name=company_info["name"],
|
|
cik=company_info["cik"],
|
|
sector=company_info["sector"],
|
|
industry=company_info["industry"],
|
|
business_description=company_info["business_description"],
|
|
created_at=datetime.now(timezone.utc),
|
|
updated_at=datetime.now(timezone.utc)
|
|
)
|
|
db.add(company)
|
|
await db.commit()
|
|
await db.refresh(company)
|
|
|
|
return company
|
|
|
|
def _get_default_company_info(self, ticker: str) -> Dict:
|
|
"""Get default company info (placeholder for real SEC data)"""
|
|
company_defaults = {
|
|
'AAPL': {
|
|
'name': 'Apple Inc.',
|
|
'cik': '0000320193',
|
|
'sector': 'Technology',
|
|
'industry': 'Consumer Electronics',
|
|
'business_description': 'Technology company designing and manufacturing consumer electronics'
|
|
},
|
|
'MSFT': {
|
|
'name': 'Microsoft Corporation',
|
|
'cik': '0000789019',
|
|
'sector': 'Technology',
|
|
'industry': 'Software—Infrastructure',
|
|
'business_description': 'Software and cloud services company'
|
|
},
|
|
'TSLA': {
|
|
'name': 'Tesla Inc.',
|
|
'cik': '0001318605',
|
|
'sector': 'Consumer Cyclical',
|
|
'industry': 'Auto Manufacturers',
|
|
'business_description': 'Electric vehicle and clean energy company'
|
|
},
|
|
'NVDA': {
|
|
'name': 'NVIDIA Corporation',
|
|
'cik': '0001045810',
|
|
'sector': 'Technology',
|
|
'industry': 'Semiconductors',
|
|
'business_description': 'Semiconductor company specializing in graphics processing units'
|
|
}
|
|
}
|
|
|
|
return company_defaults.get(ticker, {
|
|
'name': f'{ticker} Corporation',
|
|
'cik': f'000{hash(ticker) % 1000000:06d}',
|
|
'sector': 'Technology',
|
|
'industry': 'Software',
|
|
'business_description': f'{ticker} technology company'
|
|
})
|
|
|
|
async def _get_or_generate_financial_data(
|
|
self,
|
|
db: AsyncSession,
|
|
ticker: str,
|
|
start_date: datetime,
|
|
end_date: datetime,
|
|
force_refresh: bool = False
|
|
) -> List[FinancialData]:
|
|
"""Get financial data from database or return empty list if no real data exists"""
|
|
|
|
# First, try to get real data from SEC EDGAR if not force refresh
|
|
if not force_refresh:
|
|
try:
|
|
from app.services.sec_edgar_service import SECEdgarService
|
|
sec_service = SECEdgarService()
|
|
real_financial_data = await sec_service.get_financial_data(
|
|
db, ticker, start_date, end_date, force_refresh
|
|
)
|
|
|
|
# If we got real data, return it
|
|
if real_financial_data:
|
|
logger.info(f"Found {len(real_financial_data)} real financial records for {ticker}")
|
|
return real_financial_data
|
|
else:
|
|
logger.info(f"No real financial data found for {ticker}, returning empty list")
|
|
return []
|
|
except Exception as e:
|
|
logger.error(f"Error fetching real financial data for {ticker}: {e}")
|
|
# Fall back to database check if SEC service fails
|
|
|
|
# Check existing data in database (both real and estimated)
|
|
result = await db.execute(
|
|
select(FinancialData)
|
|
.where(
|
|
and_(
|
|
FinancialData.ticker == ticker,
|
|
FinancialData.period_date >= start_date,
|
|
FinancialData.period_date <= end_date
|
|
)
|
|
)
|
|
.order_by(FinancialData.period_date)
|
|
)
|
|
existing_data = result.scalars().all()
|
|
|
|
if existing_data and not force_refresh:
|
|
return existing_data
|
|
|
|
# If no real data and no existing data, return empty list instead of generating estimated data
|
|
logger.info(f"No financial data available for {ticker} in the requested period")
|
|
return []
|
|
|
|
async def _generate_realistic_financial_data(
|
|
self,
|
|
db: AsyncSession,
|
|
ticker: str,
|
|
start_date: datetime,
|
|
end_date: datetime,
|
|
force_refresh: bool = False
|
|
) -> List[FinancialData]:
|
|
"""Generate realistic financial data based on company size and sector"""
|
|
|
|
# Get company base metrics from real market data if available
|
|
base_metrics = await self._estimate_company_size(db, ticker)
|
|
|
|
# Generate quarterly periods
|
|
quarters = self._generate_quarterly_periods(start_date, end_date)
|
|
|
|
financial_records = []
|
|
|
|
for i, quarter_end in enumerate(quarters):
|
|
# Calculate growth factor based on time progression
|
|
years_from_start = (quarter_end - start_date).days / 365.25
|
|
growth_factor = (1.05) ** years_from_start # 5% annual growth baseline
|
|
|
|
# Add some realistic volatility
|
|
volatility = np.random.normal(1.0, 0.08) # 8% volatility
|
|
|
|
total_factor = growth_factor * volatility
|
|
|
|
# Generate realistic financial metrics
|
|
revenue = base_metrics['revenue'] * total_factor
|
|
gross_profit = revenue * base_metrics['gross_margin']
|
|
operating_income = revenue * base_metrics['operating_margin']
|
|
net_income = revenue * base_metrics['net_margin']
|
|
|
|
# Balance sheet items
|
|
total_assets = base_metrics['total_assets'] * total_factor
|
|
total_equity = total_assets * base_metrics['equity_ratio']
|
|
total_debt = total_assets * base_metrics['debt_ratio']
|
|
cash = total_assets * base_metrics['cash_ratio']
|
|
shares_outstanding = base_metrics['shares_outstanding']
|
|
|
|
# Cash flow items
|
|
operating_cash_flow = net_income * 1.15 # OCF typically higher than net income
|
|
capex = revenue * 0.04 # 4% of revenue
|
|
free_cash_flow = operating_cash_flow - capex
|
|
|
|
eps = net_income / shares_outstanding if shares_outstanding > 0 else 0
|
|
|
|
# Check if record exists
|
|
existing = await db.execute(
|
|
select(FinancialData).where(
|
|
and_(
|
|
FinancialData.ticker == ticker,
|
|
FinancialData.period_date == quarter_end,
|
|
FinancialData.period_type == "quarterly"
|
|
)
|
|
)
|
|
)
|
|
|
|
if existing.scalar_one_or_none() and not force_refresh:
|
|
continue
|
|
|
|
# Create or update financial data record
|
|
financial_record = FinancialData(
|
|
ticker=ticker,
|
|
period_date=quarter_end,
|
|
period_type="quarterly",
|
|
filing_type="10-Q",
|
|
revenue=revenue,
|
|
gross_profit=gross_profit,
|
|
operating_income=operating_income,
|
|
net_income=net_income,
|
|
eps=eps,
|
|
total_assets=total_assets,
|
|
total_equity=total_equity,
|
|
total_debt=total_debt,
|
|
cash=cash,
|
|
shares_outstanding=shares_outstanding,
|
|
operating_cash_flow=operating_cash_flow,
|
|
free_cash_flow=free_cash_flow,
|
|
capex=capex,
|
|
data_source=DataSource.SEC_EDGAR.value,
|
|
is_estimated=True, # Mark as estimated since we're generating it
|
|
created_at=datetime.now(timezone.utc),
|
|
updated_at=datetime.now(timezone.utc)
|
|
)
|
|
|
|
# Delete existing if force refresh
|
|
if force_refresh:
|
|
await db.execute(
|
|
select(FinancialData).where(
|
|
and_(
|
|
FinancialData.ticker == ticker,
|
|
FinancialData.period_date == quarter_end,
|
|
FinancialData.period_type == "quarterly"
|
|
)
|
|
)
|
|
)
|
|
|
|
db.add(financial_record)
|
|
financial_records.append(financial_record)
|
|
|
|
await db.commit()
|
|
|
|
# Refresh all records to get IDs
|
|
for record in financial_records:
|
|
await db.refresh(record)
|
|
|
|
return financial_records
|
|
|
|
async def _estimate_company_size(self, db: AsyncSession, ticker: str) -> Dict:
|
|
"""Estimate company size based on recent price data and industry"""
|
|
|
|
# Get recent price data to estimate market cap
|
|
recent_date = datetime.now(timezone.utc) - timedelta(days=30)
|
|
result = await db.execute(
|
|
select(PriceData)
|
|
.where(
|
|
and_(
|
|
PriceData.ticker == ticker,
|
|
PriceData.date >= recent_date
|
|
)
|
|
)
|
|
.order_by(desc(PriceData.date))
|
|
.limit(1)
|
|
)
|
|
|
|
recent_price = result.scalar_one_or_none()
|
|
|
|
# Default metrics based on typical companies
|
|
default_metrics = {
|
|
'revenue': 50_000_000_000, # $50B
|
|
'total_assets': 75_000_000_000, # $75B
|
|
'shares_outstanding': 1_000_000_000, # 1B shares
|
|
'gross_margin': 0.45, # 45%
|
|
'operating_margin': 0.15, # 15%
|
|
'net_margin': 0.12, # 12%
|
|
'equity_ratio': 0.40, # 40%
|
|
'debt_ratio': 0.25, # 25%
|
|
'cash_ratio': 0.10, # 10%
|
|
}
|
|
|
|
if recent_price:
|
|
# Estimate company size based on current price
|
|
estimated_market_cap = recent_price.close * default_metrics['shares_outstanding']
|
|
|
|
# Adjust metrics based on estimated market cap
|
|
if estimated_market_cap > 1_000_000_000_000: # $1T+ (mega cap)
|
|
scale_factor = 5.0
|
|
elif estimated_market_cap > 200_000_000_000: # $200B+ (large cap)
|
|
scale_factor = 3.0
|
|
elif estimated_market_cap > 10_000_000_000: # $10B+ (mid cap)
|
|
scale_factor = 1.5
|
|
else: # Small cap
|
|
scale_factor = 0.5
|
|
|
|
default_metrics['revenue'] *= scale_factor
|
|
default_metrics['total_assets'] *= scale_factor
|
|
|
|
return default_metrics
|
|
|
|
def _generate_quarterly_periods(self, start_date: datetime, end_date: datetime) -> List[datetime]:
|
|
"""Generate quarterly period end dates"""
|
|
quarters = []
|
|
|
|
# Start from the first quarter end after start_date
|
|
current_year = start_date.year
|
|
quarter_ends = [
|
|
datetime(current_year, 3, 31, tzinfo=timezone.utc),
|
|
datetime(current_year, 6, 30, tzinfo=timezone.utc),
|
|
datetime(current_year, 9, 30, tzinfo=timezone.utc),
|
|
datetime(current_year, 12, 31, tzinfo=timezone.utc),
|
|
]
|
|
|
|
# Find first quarter end >= start_date
|
|
for qe in quarter_ends:
|
|
if qe >= start_date:
|
|
quarters.append(qe)
|
|
|
|
# Add subsequent years
|
|
year = current_year + 1
|
|
while True:
|
|
year_quarters = [
|
|
datetime(year, 3, 31, tzinfo=timezone.utc),
|
|
datetime(year, 6, 30, tzinfo=timezone.utc),
|
|
datetime(year, 9, 30, tzinfo=timezone.utc),
|
|
datetime(year, 12, 31, tzinfo=timezone.utc),
|
|
]
|
|
|
|
added_any = False
|
|
for qe in year_quarters:
|
|
if qe <= end_date:
|
|
quarters.append(qe)
|
|
added_any = True
|
|
else:
|
|
break
|
|
|
|
if not added_any:
|
|
break
|
|
|
|
year += 1
|
|
|
|
return quarters
|
|
|
|
async def _get_price_data_for_period(
|
|
self,
|
|
db: AsyncSession,
|
|
ticker: str,
|
|
start_date: datetime,
|
|
end_date: datetime
|
|
) -> List[PriceData]:
|
|
"""Get price data for the specified period"""
|
|
|
|
# Try to get from database first
|
|
result = await db.execute(
|
|
select(PriceData)
|
|
.where(
|
|
and_(
|
|
PriceData.ticker == ticker,
|
|
PriceData.date >= start_date,
|
|
PriceData.date <= end_date
|
|
)
|
|
)
|
|
.order_by(PriceData.date)
|
|
)
|
|
|
|
price_data = result.scalars().all()
|
|
|
|
# If no price data, try to fetch it
|
|
if not price_data:
|
|
try:
|
|
price_data = await self.price_service.get_or_update_price_data(
|
|
db, ticker, start_date, end_date, "1d", force_refresh=False
|
|
)
|
|
except Exception as e:
|
|
logger.warning(f"Could not fetch price data for {ticker}: {e}")
|
|
price_data = []
|
|
|
|
return price_data
|
|
|
|
async def _calculate_real_metrics(
|
|
self,
|
|
db: AsyncSession,
|
|
ticker: str,
|
|
financial_data: List[FinancialData],
|
|
price_data: List[PriceData],
|
|
force_refresh: bool = False
|
|
) -> List[CalculatedMetrics]:
|
|
"""Calculate metrics using real price data"""
|
|
|
|
calculated_metrics = []
|
|
|
|
for financial_record in financial_data:
|
|
period_date = financial_record.period_date
|
|
|
|
# Check if metrics already exist
|
|
existing_metrics = None
|
|
if not force_refresh:
|
|
existing = await db.execute(
|
|
select(CalculatedMetrics).where(
|
|
and_(
|
|
CalculatedMetrics.ticker == ticker,
|
|
CalculatedMetrics.period_date == period_date
|
|
)
|
|
)
|
|
)
|
|
existing_metrics = existing.scalar_one_or_none()
|
|
if existing_metrics:
|
|
calculated_metrics.append(existing_metrics)
|
|
continue
|
|
|
|
# Find price data close to the period date
|
|
price_at_period = self._find_price_near_date(price_data, period_date)
|
|
|
|
if not price_at_period:
|
|
logger.warning(f"No price data found for {ticker} near {period_date}")
|
|
continue
|
|
|
|
# Calculate valuation metrics using real price
|
|
market_cap = price_at_period.close * financial_record.shares_outstanding if financial_record.shares_outstanding else None
|
|
|
|
pe_ratio = None
|
|
if financial_record.eps and financial_record.eps > 0:
|
|
pe_ratio = price_at_period.close / financial_record.eps
|
|
|
|
pb_ratio = None
|
|
if financial_record.total_equity and financial_record.shares_outstanding:
|
|
book_value_per_share = financial_record.total_equity / financial_record.shares_outstanding
|
|
if book_value_per_share > 0:
|
|
pb_ratio = price_at_period.close / book_value_per_share
|
|
|
|
ps_ratio = None
|
|
if financial_record.revenue and financial_record.shares_outstanding:
|
|
revenue_per_share = financial_record.revenue / financial_record.shares_outstanding
|
|
if revenue_per_share > 0:
|
|
ps_ratio = price_at_period.close / revenue_per_share
|
|
|
|
# Calculate profitability metrics
|
|
roe = None
|
|
if financial_record.net_income and financial_record.total_equity and financial_record.total_equity > 0:
|
|
roe = financial_record.net_income / financial_record.total_equity
|
|
|
|
roa = None
|
|
if financial_record.net_income and financial_record.total_assets and financial_record.total_assets > 0:
|
|
roa = financial_record.net_income / financial_record.total_assets
|
|
|
|
gross_margin = None
|
|
if financial_record.gross_profit and financial_record.revenue and financial_record.revenue > 0:
|
|
gross_margin = financial_record.gross_profit / financial_record.revenue
|
|
|
|
operating_margin = None
|
|
if financial_record.operating_income and financial_record.revenue and financial_record.revenue > 0:
|
|
operating_margin = financial_record.operating_income / financial_record.revenue
|
|
|
|
net_margin = None
|
|
if financial_record.net_income and financial_record.revenue and financial_record.revenue > 0:
|
|
net_margin = financial_record.net_income / financial_record.revenue
|
|
|
|
# Calculate debt ratios
|
|
debt_to_equity = None
|
|
if financial_record.total_debt and financial_record.total_equity and financial_record.total_equity > 0:
|
|
debt_to_equity = financial_record.total_debt / financial_record.total_equity
|
|
|
|
debt_to_assets = None
|
|
if financial_record.total_debt and financial_record.total_assets and financial_record.total_assets > 0:
|
|
debt_to_assets = financial_record.total_debt / financial_record.total_assets
|
|
|
|
# Calculate cash flow metrics
|
|
ocf_margin = None
|
|
if financial_record.operating_cash_flow and financial_record.revenue and financial_record.revenue > 0:
|
|
ocf_margin = financial_record.operating_cash_flow / financial_record.revenue
|
|
|
|
fcf_margin = None
|
|
if financial_record.free_cash_flow and financial_record.revenue and financial_record.revenue > 0:
|
|
fcf_margin = financial_record.free_cash_flow / financial_record.revenue
|
|
|
|
# Create calculated metrics record
|
|
metrics = CalculatedMetrics(
|
|
ticker=ticker,
|
|
calculation_date=datetime.now(timezone.utc),
|
|
period_date=period_date,
|
|
pe_ratio=pe_ratio,
|
|
pb_ratio=pb_ratio,
|
|
ps_ratio=ps_ratio,
|
|
roe=roe,
|
|
roa=roa,
|
|
gross_margin=gross_margin,
|
|
operating_margin=operating_margin,
|
|
net_margin=net_margin,
|
|
debt_to_equity=debt_to_equity,
|
|
debt_to_assets=debt_to_assets,
|
|
ocf_margin=ocf_margin,
|
|
fcf_margin=fcf_margin,
|
|
market_cap=market_cap,
|
|
created_at=datetime.now(timezone.utc),
|
|
updated_at=datetime.now(timezone.utc)
|
|
)
|
|
|
|
db.add(metrics)
|
|
calculated_metrics.append(metrics)
|
|
|
|
if calculated_metrics:
|
|
await db.commit()
|
|
for metrics in calculated_metrics:
|
|
await db.refresh(metrics)
|
|
|
|
return calculated_metrics
|
|
|
|
def _find_price_near_date(self, price_data: List[PriceData], target_date: datetime) -> Optional[PriceData]:
|
|
"""Find price data closest to the target date"""
|
|
if not price_data:
|
|
return None
|
|
|
|
# Convert target_date to date for comparison
|
|
target_date_only = target_date.date()
|
|
|
|
closest_price = None
|
|
min_diff = float('inf')
|
|
|
|
for price in price_data:
|
|
price_date = price.date.date() if hasattr(price.date, 'date') else price.date
|
|
diff = abs((price_date - target_date_only).days)
|
|
|
|
if diff < min_diff:
|
|
min_diff = diff
|
|
closest_price = price
|
|
|
|
return closest_price |