You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
639 lines
25 KiB
Python
639 lines
25 KiB
Python
"""
|
|
Real financial data service that combines price data with calculations
|
|
"""
|
|
|
|
from datetime import datetime, timezone, timedelta, date
|
|
from typing import Dict, List, Optional, Tuple, Union
|
|
import logging
|
|
import asyncio
|
|
import numpy as np
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
from sqlalchemy import select, and_, or_, desc
|
|
|
|
from app.models.financial import Company, FinancialData, CalculatedMetrics, PriceData
|
|
from app.schemas.financial import DataSource
|
|
from app.services.price_data_service import PriceDataService
|
|
from app.utils.date_utils import parse_period, quarters_to_date_range, resolve_time_parameters
|
|
from app.core.config import settings
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
class FinancialService:
|
|
"""Real financial service that uses actual price data for calculations"""
|
|
|
|
def __init__(self):
|
|
self.price_service = PriceDataService()
|
|
|
|
async def _get_ticker_max_range(self, ticker: str) -> Tuple[datetime, datetime]:
|
|
"""
|
|
Get maximum date range for a ticker by checking its listing date via yfinance_plus.
|
|
Runs the blocking yfinance call in a thread pool with a 20s timeout.
|
|
|
|
Returns:
|
|
Tuple of (listing_date, current_date) or fallback to 20 years if yfinance unavailable
|
|
"""
|
|
try:
|
|
import yfinance_plus as yf
|
|
|
|
ticker_obj = yf.Ticker(ticker)
|
|
loop = asyncio.get_event_loop()
|
|
hist = await asyncio.wait_for(
|
|
loop.run_in_executor(
|
|
None,
|
|
lambda: ticker_obj.history(period="max", interval="1mo")
|
|
),
|
|
timeout=20,
|
|
)
|
|
|
|
if not hist.empty:
|
|
earliest_date = hist.index[0].to_pydatetime()
|
|
if earliest_date.tzinfo is None:
|
|
earliest_date = earliest_date.replace(tzinfo=timezone.utc)
|
|
|
|
end_date = datetime.now(timezone.utc).replace(hour=23, minute=59, second=59, microsecond=0)
|
|
sec_start = datetime(settings.SEC_DATA_START_YEAR, 1, 1, tzinfo=timezone.utc)
|
|
actual_start = max(earliest_date, sec_start)
|
|
|
|
logger.info(f"Found actual listing date for {ticker}: {actual_start.date()}")
|
|
return actual_start, end_date
|
|
|
|
except Exception as e:
|
|
logger.warning(f"Could not get ticker info for {ticker}: {e}")
|
|
|
|
# Fallback to 20-year max if yfinance_plus fails
|
|
logger.info(f"Using fallback 20-year range for {ticker}")
|
|
end_date = datetime.now(timezone.utc).replace(hour=23, minute=59, second=59, microsecond=0)
|
|
start_date = end_date - timedelta(days=20 * 365.25) # 20 years
|
|
|
|
sec_start = datetime(settings.SEC_DATA_START_YEAR, 1, 1, tzinfo=timezone.utc)
|
|
actual_start = max(start_date, sec_start)
|
|
|
|
return actual_start, end_date
|
|
|
|
async def get_or_create_company_data(
|
|
self,
|
|
db: AsyncSession,
|
|
ticker: str,
|
|
start_date: Optional[Union[date, datetime]] = None,
|
|
end_date: Optional[Union[date, datetime]] = None,
|
|
quarters: Optional[List[str]] = None,
|
|
period: Optional[str] = None,
|
|
force_refresh: bool = False
|
|
) -> Dict:
|
|
"""
|
|
Get company data with real price-based calculations
|
|
"""
|
|
ticker = ticker.upper()
|
|
|
|
# Resolve time parameters to standard datetime range.
|
|
# _get_ticker_max_range is async, so handle "max" period before calling
|
|
# the sync resolve_time_parameters helper.
|
|
if period and period.lower() == "max":
|
|
resolved_start, resolved_end = await self._get_ticker_max_range(ticker)
|
|
else:
|
|
resolved_start, resolved_end = resolve_time_parameters(
|
|
start_date, end_date, quarters, period, ticker
|
|
)
|
|
|
|
# Get or create company — always succeeds (may have NULL sector for truly unknown tickers)
|
|
company = await self._get_or_create_company(db, ticker)
|
|
|
|
# Fetch financials and price data independently; failures return empty lists.
|
|
try:
|
|
financial_data = await self._get_or_generate_financial_data(
|
|
db, ticker, resolved_start, resolved_end, force_refresh
|
|
)
|
|
except Exception as e:
|
|
logger.warning("Financial data fetch failed for %s: %s", ticker, e)
|
|
financial_data = []
|
|
|
|
try:
|
|
price_data = await self._get_price_data_for_period(
|
|
db, ticker, resolved_start, resolved_end
|
|
)
|
|
except Exception as e:
|
|
logger.warning("Price data fetch failed for %s: %s", ticker, e)
|
|
price_data = []
|
|
|
|
# Calculate metrics using real price data
|
|
calculated_metrics = await self._calculate_real_metrics(
|
|
db, ticker, financial_data, price_data, force_refresh
|
|
)
|
|
|
|
return {
|
|
"company": company,
|
|
"financial_data": financial_data,
|
|
"calculated_metrics": calculated_metrics
|
|
}
|
|
|
|
@staticmethod
|
|
def _is_placeholder(company: Company) -> bool:
|
|
"""True when the row was created by the old hardcoded-defaults path."""
|
|
return bool(
|
|
company.sector == "Technology"
|
|
and company.industry == "Software"
|
|
and company.name
|
|
and company.name.endswith(" Corporation")
|
|
)
|
|
|
|
async def _get_or_create_company(self, db: AsyncSession, ticker: str) -> Company:
|
|
"""Get or create company record, enriching via CompanyMetadataService."""
|
|
result = await db.execute(select(Company).where(Company.ticker == ticker))
|
|
company = result.scalar_one_or_none()
|
|
# Fast path: row exists with real sector (not a hardcoded placeholder)
|
|
if company and company.sector and not self._is_placeholder(company):
|
|
return company
|
|
|
|
# Enrich via registry + yfinance
|
|
try:
|
|
from app.services import company_metadata_service as cms
|
|
meta = await cms.get_metadata(db, ticker)
|
|
except ValueError:
|
|
# Invalid ticker — still create a minimal placeholder so the rest of the
|
|
# financial pipeline doesn't break.
|
|
meta = {
|
|
"name": f"{ticker} Corporation",
|
|
"cik": None,
|
|
"exchange": None,
|
|
"sector": None,
|
|
"industry": None,
|
|
"country": None,
|
|
"market_cap": None,
|
|
"business_description": None,
|
|
}
|
|
except Exception as e:
|
|
logger.warning("CompanyMetadataService failed for %s: %s — using placeholder", ticker, e)
|
|
meta = {
|
|
"name": f"{ticker} Corporation",
|
|
"cik": None,
|
|
"exchange": None,
|
|
"sector": None,
|
|
"industry": None,
|
|
"country": None,
|
|
"market_cap": None,
|
|
"business_description": None,
|
|
}
|
|
|
|
# cms.get_metadata already upserts the Company row; re-fetch or create if missing.
|
|
result = await db.execute(select(Company).where(Company.ticker == ticker))
|
|
company = result.scalar_one_or_none()
|
|
if not company:
|
|
now = datetime.now(timezone.utc)
|
|
company = Company(
|
|
ticker=ticker,
|
|
name=meta.get("name") or f"{ticker} Corporation",
|
|
cik=meta.get("cik"),
|
|
exchange=meta.get("exchange"),
|
|
sector=meta.get("sector"),
|
|
industry=meta.get("industry"),
|
|
country=meta.get("country"),
|
|
market_cap=meta.get("market_cap"),
|
|
business_description=meta.get("business_description"),
|
|
created_at=now,
|
|
updated_at=now,
|
|
)
|
|
db.add(company)
|
|
await db.commit()
|
|
await db.refresh(company)
|
|
|
|
return company
|
|
|
|
async def _get_or_generate_financial_data(
|
|
self,
|
|
db: AsyncSession,
|
|
ticker: str,
|
|
start_date: datetime,
|
|
end_date: datetime,
|
|
force_refresh: bool = False
|
|
) -> List[FinancialData]:
|
|
"""Get financial data from database or return empty list if no real data exists"""
|
|
|
|
# First, try to get real data from SEC EDGAR if not force refresh
|
|
if not force_refresh:
|
|
try:
|
|
from app.services.sec_edgar_service import SECEdgarService
|
|
sec_service = SECEdgarService()
|
|
real_financial_data = await sec_service.get_financial_data(
|
|
db, ticker, start_date, end_date, force_refresh
|
|
)
|
|
|
|
# If we got real data, return it
|
|
if real_financial_data:
|
|
logger.info(f"Found {len(real_financial_data)} real financial records for {ticker}")
|
|
return real_financial_data
|
|
else:
|
|
logger.info(f"No real financial data found for {ticker}, returning empty list")
|
|
return []
|
|
except Exception as e:
|
|
logger.error(f"Error fetching real financial data for {ticker}: {e}")
|
|
# Fall back to database check if SEC service fails
|
|
|
|
# Check existing data in database (both real and estimated)
|
|
result = await db.execute(
|
|
select(FinancialData)
|
|
.where(
|
|
and_(
|
|
FinancialData.ticker == ticker,
|
|
FinancialData.period_date >= start_date,
|
|
FinancialData.period_date <= end_date
|
|
)
|
|
)
|
|
.order_by(FinancialData.period_date)
|
|
)
|
|
existing_data = result.scalars().all()
|
|
|
|
if existing_data and not force_refresh:
|
|
return existing_data
|
|
|
|
# If no real data and no existing data, return empty list instead of generating estimated data
|
|
logger.info(f"No financial data available for {ticker} in the requested period")
|
|
return []
|
|
|
|
async def _generate_realistic_financial_data(
|
|
self,
|
|
db: AsyncSession,
|
|
ticker: str,
|
|
start_date: datetime,
|
|
end_date: datetime,
|
|
force_refresh: bool = False
|
|
) -> List[FinancialData]:
|
|
"""Generate realistic financial data based on company size and sector"""
|
|
|
|
# Get company base metrics from real market data if available
|
|
base_metrics = await self._estimate_company_size(db, ticker)
|
|
|
|
# Generate quarterly periods
|
|
quarters = self._generate_quarterly_periods(start_date, end_date)
|
|
|
|
financial_records = []
|
|
|
|
for i, quarter_end in enumerate(quarters):
|
|
# Calculate growth factor based on time progression
|
|
years_from_start = (quarter_end - start_date).days / 365.25
|
|
growth_factor = (1.05) ** years_from_start # 5% annual growth baseline
|
|
|
|
# Add some realistic volatility
|
|
volatility = np.random.normal(1.0, 0.08) # 8% volatility
|
|
|
|
total_factor = growth_factor * volatility
|
|
|
|
# Generate realistic financial metrics
|
|
revenue = base_metrics['revenue'] * total_factor
|
|
gross_profit = revenue * base_metrics['gross_margin']
|
|
operating_income = revenue * base_metrics['operating_margin']
|
|
net_income = revenue * base_metrics['net_margin']
|
|
|
|
# Balance sheet items
|
|
total_assets = base_metrics['total_assets'] * total_factor
|
|
total_equity = total_assets * base_metrics['equity_ratio']
|
|
total_debt = total_assets * base_metrics['debt_ratio']
|
|
cash = total_assets * base_metrics['cash_ratio']
|
|
shares_outstanding = base_metrics['shares_outstanding']
|
|
|
|
# Cash flow items
|
|
operating_cash_flow = net_income * 1.15 # OCF typically higher than net income
|
|
capex = revenue * 0.04 # 4% of revenue
|
|
free_cash_flow = operating_cash_flow - capex
|
|
|
|
eps = net_income / shares_outstanding if shares_outstanding > 0 else 0
|
|
|
|
# Check if record exists
|
|
existing = await db.execute(
|
|
select(FinancialData).where(
|
|
and_(
|
|
FinancialData.ticker == ticker,
|
|
FinancialData.period_date == quarter_end,
|
|
FinancialData.period_type == "quarterly"
|
|
)
|
|
)
|
|
)
|
|
|
|
if existing.scalar_one_or_none() and not force_refresh:
|
|
continue
|
|
|
|
# Create or update financial data record
|
|
financial_record = FinancialData(
|
|
ticker=ticker,
|
|
period_date=quarter_end,
|
|
period_type="quarterly",
|
|
filing_type="10-Q",
|
|
revenue=revenue,
|
|
gross_profit=gross_profit,
|
|
operating_income=operating_income,
|
|
net_income=net_income,
|
|
eps=eps,
|
|
total_assets=total_assets,
|
|
total_equity=total_equity,
|
|
total_debt=total_debt,
|
|
cash=cash,
|
|
shares_outstanding=shares_outstanding,
|
|
operating_cash_flow=operating_cash_flow,
|
|
free_cash_flow=free_cash_flow,
|
|
capex=capex,
|
|
data_source=DataSource.SEC_EDGAR.value,
|
|
is_estimated=True, # Mark as estimated since we're generating it
|
|
created_at=datetime.now(timezone.utc),
|
|
updated_at=datetime.now(timezone.utc)
|
|
)
|
|
|
|
# Delete existing if force refresh
|
|
if force_refresh:
|
|
await db.execute(
|
|
select(FinancialData).where(
|
|
and_(
|
|
FinancialData.ticker == ticker,
|
|
FinancialData.period_date == quarter_end,
|
|
FinancialData.period_type == "quarterly"
|
|
)
|
|
)
|
|
)
|
|
|
|
db.add(financial_record)
|
|
financial_records.append(financial_record)
|
|
|
|
await db.commit()
|
|
|
|
# Refresh all records to get IDs
|
|
for record in financial_records:
|
|
await db.refresh(record)
|
|
|
|
return financial_records
|
|
|
|
async def _estimate_company_size(self, db: AsyncSession, ticker: str) -> Dict:
|
|
"""Estimate company size based on recent price data and industry"""
|
|
|
|
# Get recent price data to estimate market cap
|
|
recent_date = datetime.now(timezone.utc) - timedelta(days=30)
|
|
result = await db.execute(
|
|
select(PriceData)
|
|
.where(
|
|
and_(
|
|
PriceData.ticker == ticker,
|
|
PriceData.date >= recent_date
|
|
)
|
|
)
|
|
.order_by(desc(PriceData.date))
|
|
.limit(1)
|
|
)
|
|
|
|
recent_price = result.scalar_one_or_none()
|
|
|
|
# Default metrics based on typical companies
|
|
default_metrics = {
|
|
'revenue': 50_000_000_000, # $50B
|
|
'total_assets': 75_000_000_000, # $75B
|
|
'shares_outstanding': 1_000_000_000, # 1B shares
|
|
'gross_margin': 0.45, # 45%
|
|
'operating_margin': 0.15, # 15%
|
|
'net_margin': 0.12, # 12%
|
|
'equity_ratio': 0.40, # 40%
|
|
'debt_ratio': 0.25, # 25%
|
|
'cash_ratio': 0.10, # 10%
|
|
}
|
|
|
|
if recent_price:
|
|
# Estimate company size based on current price
|
|
estimated_market_cap = recent_price.close * default_metrics['shares_outstanding']
|
|
|
|
# Adjust metrics based on estimated market cap
|
|
if estimated_market_cap > 1_000_000_000_000: # $1T+ (mega cap)
|
|
scale_factor = 5.0
|
|
elif estimated_market_cap > 200_000_000_000: # $200B+ (large cap)
|
|
scale_factor = 3.0
|
|
elif estimated_market_cap > 10_000_000_000: # $10B+ (mid cap)
|
|
scale_factor = 1.5
|
|
else: # Small cap
|
|
scale_factor = 0.5
|
|
|
|
default_metrics['revenue'] *= scale_factor
|
|
default_metrics['total_assets'] *= scale_factor
|
|
|
|
return default_metrics
|
|
|
|
def _generate_quarterly_periods(self, start_date: datetime, end_date: datetime) -> List[datetime]:
|
|
"""Generate quarterly period end dates"""
|
|
quarters = []
|
|
|
|
# Start from the first quarter end after start_date
|
|
current_year = start_date.year
|
|
quarter_ends = [
|
|
datetime(current_year, 3, 31, tzinfo=timezone.utc),
|
|
datetime(current_year, 6, 30, tzinfo=timezone.utc),
|
|
datetime(current_year, 9, 30, tzinfo=timezone.utc),
|
|
datetime(current_year, 12, 31, tzinfo=timezone.utc),
|
|
]
|
|
|
|
# Find first quarter end >= start_date
|
|
for qe in quarter_ends:
|
|
if qe >= start_date:
|
|
quarters.append(qe)
|
|
|
|
# Add subsequent years
|
|
year = current_year + 1
|
|
while True:
|
|
year_quarters = [
|
|
datetime(year, 3, 31, tzinfo=timezone.utc),
|
|
datetime(year, 6, 30, tzinfo=timezone.utc),
|
|
datetime(year, 9, 30, tzinfo=timezone.utc),
|
|
datetime(year, 12, 31, tzinfo=timezone.utc),
|
|
]
|
|
|
|
added_any = False
|
|
for qe in year_quarters:
|
|
if qe <= end_date:
|
|
quarters.append(qe)
|
|
added_any = True
|
|
else:
|
|
break
|
|
|
|
if not added_any:
|
|
break
|
|
|
|
year += 1
|
|
|
|
return quarters
|
|
|
|
async def _get_price_data_for_period(
|
|
self,
|
|
db: AsyncSession,
|
|
ticker: str,
|
|
start_date: datetime,
|
|
end_date: datetime
|
|
) -> List[PriceData]:
|
|
"""Get price data for the specified period"""
|
|
|
|
# Try to get from database first
|
|
result = await db.execute(
|
|
select(PriceData)
|
|
.where(
|
|
and_(
|
|
PriceData.ticker == ticker,
|
|
PriceData.date >= start_date,
|
|
PriceData.date <= end_date
|
|
)
|
|
)
|
|
.order_by(PriceData.date)
|
|
)
|
|
|
|
price_data = result.scalars().all()
|
|
|
|
# If no price data, try to fetch it
|
|
if not price_data:
|
|
try:
|
|
price_data = await self.price_service.get_or_update_price_data(
|
|
ticker, start_date, end_date, "1d", force_refresh=False
|
|
)
|
|
except Exception as e:
|
|
logger.warning(f"Could not fetch price data for {ticker}: {e}")
|
|
price_data = []
|
|
|
|
return price_data
|
|
|
|
async def _calculate_real_metrics(
|
|
self,
|
|
db: AsyncSession,
|
|
ticker: str,
|
|
financial_data: List[FinancialData],
|
|
price_data: List[PriceData],
|
|
force_refresh: bool = False
|
|
) -> List[CalculatedMetrics]:
|
|
"""Calculate metrics using real price data"""
|
|
|
|
calculated_metrics = []
|
|
|
|
for financial_record in financial_data:
|
|
period_date = financial_record.period_date
|
|
|
|
# Check if metrics already exist
|
|
existing_metrics = None
|
|
if not force_refresh:
|
|
existing = await db.execute(
|
|
select(CalculatedMetrics).where(
|
|
and_(
|
|
CalculatedMetrics.ticker == ticker,
|
|
CalculatedMetrics.period_date == period_date
|
|
)
|
|
)
|
|
)
|
|
existing_metrics = existing.scalar_one_or_none()
|
|
if existing_metrics:
|
|
calculated_metrics.append(existing_metrics)
|
|
continue
|
|
|
|
# Find price data close to the period date
|
|
price_at_period = self._find_price_near_date(price_data, period_date)
|
|
|
|
if not price_at_period:
|
|
logger.warning(f"No price data found for {ticker} near {period_date}")
|
|
continue
|
|
|
|
# Calculate valuation metrics using real price
|
|
market_cap = price_at_period.close * financial_record.shares_outstanding if financial_record.shares_outstanding else None
|
|
|
|
pe_ratio = None
|
|
if financial_record.eps and financial_record.eps > 0:
|
|
pe_ratio = price_at_period.close / financial_record.eps
|
|
|
|
pb_ratio = None
|
|
if financial_record.total_equity and financial_record.shares_outstanding:
|
|
book_value_per_share = financial_record.total_equity / financial_record.shares_outstanding
|
|
if book_value_per_share > 0:
|
|
pb_ratio = price_at_period.close / book_value_per_share
|
|
|
|
ps_ratio = None
|
|
if financial_record.revenue and financial_record.shares_outstanding:
|
|
revenue_per_share = financial_record.revenue / financial_record.shares_outstanding
|
|
if revenue_per_share > 0:
|
|
ps_ratio = price_at_period.close / revenue_per_share
|
|
|
|
# Calculate profitability metrics
|
|
roe = None
|
|
if financial_record.net_income and financial_record.total_equity and financial_record.total_equity > 0:
|
|
roe = financial_record.net_income / financial_record.total_equity
|
|
|
|
roa = None
|
|
if financial_record.net_income and financial_record.total_assets and financial_record.total_assets > 0:
|
|
roa = financial_record.net_income / financial_record.total_assets
|
|
|
|
gross_margin = None
|
|
if financial_record.gross_profit and financial_record.revenue and financial_record.revenue > 0:
|
|
gross_margin = financial_record.gross_profit / financial_record.revenue
|
|
|
|
operating_margin = None
|
|
if financial_record.operating_income and financial_record.revenue and financial_record.revenue > 0:
|
|
operating_margin = financial_record.operating_income / financial_record.revenue
|
|
|
|
net_margin = None
|
|
if financial_record.net_income and financial_record.revenue and financial_record.revenue > 0:
|
|
net_margin = financial_record.net_income / financial_record.revenue
|
|
|
|
# Calculate debt ratios
|
|
debt_to_equity = None
|
|
if financial_record.total_debt and financial_record.total_equity and financial_record.total_equity > 0:
|
|
debt_to_equity = financial_record.total_debt / financial_record.total_equity
|
|
|
|
debt_to_assets = None
|
|
if financial_record.total_debt and financial_record.total_assets and financial_record.total_assets > 0:
|
|
debt_to_assets = financial_record.total_debt / financial_record.total_assets
|
|
|
|
# Calculate cash flow metrics
|
|
ocf_margin = None
|
|
if financial_record.operating_cash_flow and financial_record.revenue and financial_record.revenue > 0:
|
|
ocf_margin = financial_record.operating_cash_flow / financial_record.revenue
|
|
|
|
fcf_margin = None
|
|
if financial_record.free_cash_flow and financial_record.revenue and financial_record.revenue > 0:
|
|
fcf_margin = financial_record.free_cash_flow / financial_record.revenue
|
|
|
|
# Create calculated metrics record
|
|
metrics = CalculatedMetrics(
|
|
ticker=ticker,
|
|
calculation_date=datetime.now(timezone.utc),
|
|
period_date=period_date,
|
|
pe_ratio=pe_ratio,
|
|
pb_ratio=pb_ratio,
|
|
ps_ratio=ps_ratio,
|
|
roe=roe,
|
|
roa=roa,
|
|
gross_margin=gross_margin,
|
|
operating_margin=operating_margin,
|
|
net_margin=net_margin,
|
|
debt_to_equity=debt_to_equity,
|
|
debt_to_assets=debt_to_assets,
|
|
ocf_margin=ocf_margin,
|
|
fcf_margin=fcf_margin,
|
|
market_cap=market_cap,
|
|
created_at=datetime.now(timezone.utc),
|
|
updated_at=datetime.now(timezone.utc)
|
|
)
|
|
|
|
db.add(metrics)
|
|
calculated_metrics.append(metrics)
|
|
|
|
if calculated_metrics:
|
|
await db.commit()
|
|
for metrics in calculated_metrics:
|
|
await db.refresh(metrics)
|
|
|
|
return calculated_metrics
|
|
|
|
def _find_price_near_date(self, price_data: List[PriceData], target_date: datetime) -> Optional[PriceData]:
|
|
"""Find price data closest to the target date"""
|
|
if not price_data:
|
|
return None
|
|
|
|
# Convert target_date to date for comparison
|
|
target_date_only = target_date.date()
|
|
|
|
closest_price = None
|
|
min_diff = float('inf')
|
|
|
|
for price in price_data:
|
|
price_date = price.date.date() if hasattr(price.date, 'date') else price.date
|
|
diff = abs((price_date - target_date_only).days)
|
|
|
|
if diff < min_diff:
|
|
min_diff = diff
|
|
closest_price = price
|
|
|
|
return closest_price |