stock-oracle/app/api/v1/endpoints/financial.py

"""
Financial data endpoints
"""

from datetime import datetime, timezone, date
from typing import List, Optional
from fastapi import APIRouter, Depends, HTTPException, Query, Response
from sqlalchemy.ext.asyncio import AsyncSession

from app.core.database import get_db, AsyncSessionLocal
from app.schemas.financial import (
    FinancialDataRequest,
    FinancialDataResponse,
    BulkFinancialDataRequest,
    BulkFinancialDataResponse,
    BulkFinancialDataItem,
    ErrorResponse,
    ErrorType,
    CompanyInfo,
    FinancialDataPoint,
    CalculatedMetricsData
)
from app.services.sec_data_service import SECDataService
from app.core.config import settings
from app.utils.date_utils import quarters_to_date_range
from app.utils.cache import (
    build_cache_key,
    get_cached_response,
    set_cached_response,
)

router = APIRouter()

@router.post(
    "/data",
    response_model=FinancialDataResponse,
    responses={
        400: {"model": ErrorResponse, "description": "Invalid request parameters"},
        404: {"model": ErrorResponse, "description": "Data not found"},
        500: {"model": ErrorResponse, "description": "Internal server error"}
    },
    summary="Get SEC EDGAR financial data for a ticker",
    description="""
    Retrieve comprehensive financial data directly from SEC EDGAR filings for a specific ticker and time period.
    
    **🔥 Three Ways to Specify Time Period (choose one):**
    
    1. **Period String** (NEW! Most convenient):
       - `period`: "1d", "7d", "30d", "1m", "3m", "6m", "1y", "2y", "5y", "max"
       - Examples: `{"ticker": "AAPL", "period": "1y"}` - Last 1 year of data
       - Example: `{"ticker": "TSLA", "period": "max"}` - All available data from listing date to SEC limits
       
    2. **Date Range** (Traditional):
       - `start_date` + `end_date`: Specific date range
       - Example: `{"ticker": "AAPL", "start_date": "2024-01-01", "end_date": "2024-12-31"}`
       
    3. **Quarters** (Quarter-based):
       - `quarters`: List of quarters like ["2024Q1", "2024Q2"]
       - Example: `{"ticker": "AAPL", "quarters": ["2024Q1", "2024Q2", "2024Q3"]}`
    
    **Data Sources:**
    - **Financial Data**: Direct SEC EDGAR API calls (revenue, income, assets, cash flow)
    - **Price Data**: Available via separate price data endpoints using yfinance-plus
    
    **This endpoint returns:**
    - Company information (name, CIK, sector, industry)
    - Financial statements data from SEC filings (income statement, balance sheet, cash flow)
    - Calculated financial metrics (ratios, margins, growth rates)
    - Period types: quarterly (10-Q) and annual (10-K) filings
    
    **Performance Features:**
    - Database caching to avoid repeated SEC API calls
    - Historical data available from 1994-present
    - 15+ years of data typically available for most companies
    - Use `force_refresh=true` to fetch fresh data from SEC EDGAR
    
    **Data Quality:**
    - All financial data sourced directly from official SEC filings
    - No estimated or synthetic data - only actual reported figures
    - Automatic validation and error handling for missing periods
    
    **Example Requests:**
    ```json
    // Using period (simplest)
    {
        "ticker": "AAPL",
        "period": "1y",
        "include_metrics": true
    }
    
    // Using date range
    {
        "ticker": "MSFT",
        "start_date": "2024-01-01",
        "end_date": "2024-12-31",
        "period_type": "quarterly"
    }
    
    // Using quarters
    {
        "ticker": "GOOGL",
        "quarters": ["2024Q1", "2024Q2"],
        "include_metrics": true
    }
    ```
    """
)
async def get_financial_data(
    request: FinancialDataRequest,
    response: Response,
    db: AsyncSession = Depends(get_db)
):
    """Get financial data for a ticker using period, quarters, or date range"""
    
    try:
        # Use the updated service that handles period resolution
        from app.services.financial_service import FinancialService
        financial_service = FinancialService()
        
        # Resolve time parameters for metadata
        from app.utils.date_utils import resolve_time_parameters
        resolved_start, resolved_end = resolve_time_parameters(
            request.start_date, request.end_date, request.quarters, request.period, request.ticker,
            ticker_max_range_fn=financial_service._get_ticker_max_range
        )

        # Build cache key using normalized inputs
        cache_key = build_cache_key(
            "financial:data",
            request.ticker.upper(),
            request.period_type.value if hasattr(request.period_type, 'value') else str(request.period_type),
            "metrics" if request.include_metrics else "no-metrics",
            (resolved_start.date().isoformat() if resolved_start else ""),
            (resolved_end.date().isoformat() if resolved_end else ""),
        )

        # Financial data is released quarterly — use 24 h TTL
        _FIN_TTL = 60 * 60 * 24

        # Try cache unless force_refresh
        if not request.force_refresh:
            cached = await get_cached_response(cache_key)
            if cached:
                cached_body, etag = cached
                response.headers["X-Cache"] = "HIT"
                response.headers["Cache-Control"] = f"public, max-age={_FIN_TTL}"
                response.headers["ETag"] = etag
                return cached_body
        
        data = await financial_service.get_or_create_company_data(
            db,
            request.ticker,
            start_date=request.start_date,
            end_date=request.end_date,
            quarters=request.quarters,
            period=request.period,
            force_refresh=request.force_refresh
        )
        
        # Format response
        company = data["company"]
        financial_data = data["financial_data"]
        calculated_metrics = data["calculated_metrics"]
        
        # Convert to response models
        company_info = CompanyInfo(
            ticker=company.ticker,
            name=company.name,
            cik=company.cik,
            sector=company.sector,
            industry=company.industry,
            business_description=company.business_description
        )
        
        # Filter financial data by period type
        if request.period_type != "all":
            financial_data = [fd for fd in financial_data if fd.period_type == request.period_type]
        
        # Merge financial data with calculated metrics
        financial_points = []
        for fd in financial_data:
            # Convert to dict for merging
            fd_dict = fd.__dict__ if hasattr(fd, '__dict__') else {}
            
            # Find matching calculated metrics for this period
            matching_metrics = None
            if request.include_metrics and calculated_metrics:
                for cm in calculated_metrics:
                    if cm.period_date == fd.period_date:
                        matching_metrics = cm
                        break
            
            # Merge metrics into financial data point
            if matching_metrics:
                fd_dict.update({
                    'pe_ratio': matching_metrics.pe_ratio,
                    'pb_ratio': matching_metrics.pb_ratio,
                    'ps_ratio': matching_metrics.ps_ratio,
                    'roe': matching_metrics.roe,
                    'roa': matching_metrics.roa,
                    'gross_margin': matching_metrics.gross_margin,
                    'operating_margin': matching_metrics.operating_margin,
                    'net_margin': matching_metrics.net_margin,
                    'debt_to_equity': matching_metrics.debt_to_equity,
                    'debt_to_assets': matching_metrics.debt_to_assets,
                    'ocf_margin': matching_metrics.ocf_margin,
                    'fcf_margin': matching_metrics.fcf_margin,
                    'market_cap': matching_metrics.market_cap
                })
            
            financial_points.append(FinancialDataPoint.model_validate(fd_dict))
        
        # Calculate actual date range from returned data
        actual_start_date = resolved_start
        actual_end_date = resolved_end
        
        if financial_points:
            # Get actual start and end dates from the financial data
            actual_start_date = min(point.period_date for point in financial_points)
            actual_end_date = max(point.period_date for point in financial_points)
        
        body = FinancialDataResponse(
            company=company_info,
            financial_data=financial_points,
            metadata={
                "request_id": str(request.ticker),
                "data_points": len(financial_points),
                "period_type": request.period_type.value,
                "quarters_requested": request.quarters if request.quarters else None,
                "date_range": {
                    "start": actual_start_date.isoformat(),
                    "end": actual_end_date.isoformat()
                },
                "last_updated": datetime.now(timezone.utc).isoformat()
            }
        )

        etag = await set_cached_response(cache_key, body.model_dump(), ttl_seconds=_FIN_TTL)
        response.headers["X-Cache"] = "MISS"
        response.headers["Cache-Control"] = f"public, max-age={_FIN_TTL}"
        response.headers["ETag"] = etag
        return body
        
    except HTTPException:
        raise
    except ValueError as e:
        if "No data returned" in str(e):
            raise HTTPException(
                status_code=404,
                detail={
                    "error_type": ErrorType.DATA_NOT_FOUND,
                    "message": f"No financial data found for ticker {request.ticker}",
                    "detail": {
                        "ticker": request.ticker,
                        "period": f"{request.start_date} to {request.end_date}"
                    }
                }
            )
        raise HTTPException(
            status_code=400,
            detail={
                "error_type": ErrorType.PARSING_ERROR,
                "message": str(e)
            }
        )
    except Exception as e:
        raise HTTPException(
            status_code=500,
            detail={
                "error_type": ErrorType.DATABASE_ERROR,
                "message": "An error occurred while processing your request",
                "detail": {"error": str(e)}
            }
        )

@router.get(
    "/data/{ticker}",
    response_model=FinancialDataResponse,
    summary="Get financial data by ticker (simplified)",
    description="""
    Simplified GET endpoint to retrieve financial data with query parameters.
    
    **Time Period Options:**
    - Use `period` for convenience: "1d", "7d", "1m", "3m", "6m", "1y", "2y", "5y", "max"
    - OR use `start_date` and `end_date` for specific date range
    - Cannot use both approaches simultaneously
    
    **Examples:**
    - `/api/v1/financial/data/AAPL?period=1y&include_metrics=true` - Last year of financial data
    - `/api/v1/financial/data/AAPL?start_date=2024-01-01&end_date=2024-12-31&period_type=quarterly` - Specific date range
    """
)
async def get_financial_data_simple(
    ticker: str,
    response: Response,
    period: Optional[str] = Query(None, description="Period like '1d', '7d', '1m', '3m', '6m', '1y', '2y', '5y', 'max'"),
    start_date: Optional[date] = Query(None, description="Start date for data retrieval (use with end_date, not with period)"),
    end_date: Optional[date] = Query(None, description="End date for data retrieval (use with start_date, not with period)"),
    period_type: str = Query("all", description="Period type: quarterly, annual, or all"),
    include_metrics: bool = Query(True, description="Include calculated metrics"),
    force_refresh: bool = Query(False, description="Force refresh from SEC"),
    db: AsyncSession = Depends(get_db)
):
    """Simplified GET endpoint for financial data"""
    # Validate that either period OR date range is provided, not both
    if period and (start_date or end_date):
        raise HTTPException(
            status_code=400,
            detail={
                "error_type": ErrorType.VALIDATION_ERROR,
                "message": "Cannot specify both period and date range. Use either period OR start_date+end_date."
            }
        )
    
    if not period and not (start_date and end_date):
        raise HTTPException(
            status_code=400,
            detail={
                "error_type": ErrorType.VALIDATION_ERROR,
                "message": "Must specify either period OR both start_date and end_date."
            }
        )
    
    # Create request based on provided parameters
    if period:
        request = FinancialDataRequest(
            ticker=ticker,
            period=period,
            period_type=period_type,
            include_metrics=include_metrics,
            force_refresh=force_refresh
        )
    else:
        request = FinancialDataRequest(
            ticker=ticker,
            start_date=start_date,
            end_date=end_date,
            period_type=period_type,
            include_metrics=include_metrics,
            force_refresh=force_refresh
        )
    
    return await get_financial_data(request, response, db)

@router.post(
    "/data/bulk",
    response_model=BulkFinancialDataResponse,
    responses={
        400: {"model": ErrorResponse, "description": "Invalid request parameters"},
        500: {"model": ErrorResponse, "description": "Internal server error"}
    },
    summary="Get SEC EDGAR financial data for multiple tickers",
    description="""
    Retrieve comprehensive financial data for multiple tickers in a single request directly from SEC EDGAR filings.
    
    **🔥 Three Ways to Specify Time Period (choose one):**
    
    1. **Period String** (NEW! Most convenient):
       - `period`: "1d", "7d", "30d", "1m", "3m", "6m", "1y", "2y", "5y", "max"
       - Example: Last 1 year for multiple tickers, or "max" for all available data
       
    2. **Date Range** (Traditional):
       - `start_date` + `end_date`: Specific date range
       - Example: Specific date range for all tickers
       
    3. **Quarters** (Quarter-based):
       - `quarters`: List of quarters like ["2024Q1", "2024Q2"]
       - Example: Specific quarters for all tickers
    
    **Data Sources:**
    - **Financial Data**: Direct SEC EDGAR API calls (revenue, income, assets, cash flow)
    - **Price Data**: Available via separate price data endpoints using yfinance-plus
    
    **Bulk Processing Features:**
    - Processes up to 100 tickers in parallel for maximum efficiency
    - Returns individual success/failure results for each ticker
    - Handles partial failures gracefully (some tickers can fail while others succeed)
    - Uses the same robust SEC data retrieval logic as single ticker endpoint
    
    **SEC EDGAR Integration:**
    - Direct API calls to official SEC EDGAR database
    - All financial data sourced from actual SEC filings (10-K, 10-Q)
    - No estimated or synthetic data - only actual reported figures
    - Historical data available from 1994-present (15+ years for most companies)
    - Automatic validation and error handling for missing periods
    
    **Data Quality & Features:**
    - Company information (name, CIK, sector, industry, business description)
    - Comprehensive financial statements (income statement, balance sheet, cash flow)
    - Calculated financial metrics (ratios, margins, growth rates)
    - Period types: quarterly (10-Q) and annual (10-K) filings
    - Database caching to avoid repeated SEC API calls
    
    **Performance:**
    - Parallel processing for bulk requests
    - Intelligent caching and rate limiting
    - Use `force_refresh=true` to fetch fresh data from SEC EDGAR
    
    **Example Requests:**
    ```json
    // Using period (simplest)
    {
        "tickers": ["AAPL", "MSFT", "GOOGL"],
        "period": "1y",
        "include_metrics": true
    }
    
    // Using date range
    {
        "tickers": ["NVDA", "AMD", "INTC"],
        "start_date": "2024-01-01",
        "end_date": "2024-12-31",
        "period_type": "quarterly"
    }
    
    // Using quarters
    {
        "tickers": ["TSLA", "F", "GM"],
        "quarters": ["2024Q1", "2024Q2"],
        "include_metrics": true
    }
    ```
    
    Each ticker result includes the same comprehensive financial data structure as the single ticker endpoint.
    Failed tickers will have detailed error messages while successful ones will have complete SEC filing data.
    """
)
async def get_bulk_financial_data(
    request: BulkFinancialDataRequest,
    db: AsyncSession = Depends(get_db)
):
    """Get financial data for multiple tickers"""
    
    # Convert time parameters to dates using the same logic as single endpoint
    if request.period:
        # Use period approach - import the parse_period function
        from app.utils.date_utils import parse_period
        try:
            start_date, end_date = parse_period(request.period)
        except ValueError as e:
            raise HTTPException(
                status_code=400,
                detail={
                    "error_type": ErrorType.VALIDATION_ERROR,
                    "message": f"Invalid period format: {str(e)}"
                }
            )
    elif request.quarters:
        try:
            start_date, end_date = quarters_to_date_range(request.quarters)
        except ValueError as e:
            raise HTTPException(
                status_code=400,
                detail={
                    "error_type": ErrorType.VALIDATION_ERROR,
                    "message": str(e)
                }
            )
    else:
        # Convert date to datetime for internal processing
        start_date = datetime.combine(request.start_date, datetime.min.time()).replace(tzinfo=timezone.utc) if request.start_date else None
        end_date = datetime.combine(request.end_date, datetime.max.time()).replace(tzinfo=timezone.utc) if request.end_date else None
    
    # Validate date range
    if start_date and end_date and start_date >= end_date:
        raise HTTPException(
            status_code=400,
            detail={
                "error_type": ErrorType.VALIDATION_ERROR,
                "message": "Start date must be before end date"
            }
        )
    
    # Check if requested period is valid (SEC data available from 1994)
    if start_date and start_date.year < settings.SEC_DATA_START_YEAR:
        raise HTTPException(
            status_code=400,
            detail={
                "error_type": ErrorType.INVALID_PERIOD,
                "message": f"SEC data is only available from {settings.SEC_DATA_START_YEAR}",
                "detail": {
                    "requested_start": start_date.isoformat(),
                    "earliest_available": f"{settings.SEC_DATA_START_YEAR}-01-01"
                }
            }
        )
    
    # Future date check
    current_time = datetime.now(timezone.utc)
    if start_date and start_date > current_time:
        raise HTTPException(
            status_code=400,
            detail={
                "error_type": ErrorType.INVALID_PERIOD,
                "message": "Cannot request data for future dates"
            }
        )
    
    import asyncio
    
    results = []
    successful_count = 0
    failed_count = 0
    
    async def process_ticker(ticker: str):
        """Process a single ticker and return result"""
        try:
            # Use an isolated DB session per task to avoid concurrent use of a single session
            async with AsyncSessionLocal() as session:
                # Get data using existing service
                sec_service = SECDataService()
                data = await sec_service.get_or_update_company_data(
                    session,
                    ticker,
                    start_date,
                    end_date,
                    request.force_refresh
                )
            
            # Format response
            company = data["company"]
            financial_data = data["financial_data"]
            calculated_metrics = data["calculated_metrics"]  # Always include metrics
            
            # Convert to response models
            company_info = CompanyInfo(
                ticker=company.ticker,
                name=company.name,
                cik=company.cik,
                sector=company.sector,
                industry=company.industry,
                business_description=company.business_description
            )
            
            # Filter financial data by period type
            if request.period_type != "all":
                financial_data = [fd for fd in financial_data if fd.period_type == request.period_type]
            
            # Merge financial data with calculated metrics
            financial_points = []
            for fd in financial_data:
                # Convert to dict for merging
                fd_dict = fd.__dict__ if hasattr(fd, '__dict__') else {}
                
                # Find matching calculated metrics for this period
                matching_metrics = None
                if request.include_metrics and calculated_metrics:
                    for cm in calculated_metrics:
                        if cm.period_date == fd.period_date:
                            matching_metrics = cm
                            break
                
                # Merge metrics into financial data point
                if matching_metrics:
                    fd_dict.update({
                        'pe_ratio': matching_metrics.pe_ratio,
                        'pb_ratio': matching_metrics.pb_ratio,
                        'ps_ratio': matching_metrics.ps_ratio,
                        'roe': matching_metrics.roe,
                        'roa': matching_metrics.roa,
                        'gross_margin': matching_metrics.gross_margin,
                        'operating_margin': matching_metrics.operating_margin,
                        'net_margin': matching_metrics.net_margin,
                        'debt_to_equity': matching_metrics.debt_to_equity,
                        'debt_to_assets': matching_metrics.debt_to_assets,
                        'ocf_margin': matching_metrics.ocf_margin,
                        'fcf_margin': matching_metrics.fcf_margin,
                        'market_cap': matching_metrics.market_cap
                    })
                
                financial_points.append(FinancialDataPoint.model_validate(fd_dict))
            
            # Calculate actual date range from returned data
            actual_start_date = start_date
            actual_end_date = end_date
            
            if financial_points:
                # Get actual start and end dates from the financial data
                actual_start_date = min(point.period_date for point in financial_points)
                actual_end_date = max(point.period_date for point in financial_points)
            
            response = FinancialDataResponse(
                company=company_info,
                financial_data=financial_points,
                metadata={
                    "request_id": str(ticker),
                    "data_points": len(financial_points),
                    "period_type": request.period_type.value,
                    "quarters_requested": request.quarters if request.quarters else None,
                    "date_range": {
                        "start": actual_start_date.isoformat(),
                        "end": actual_end_date.isoformat()
                    },
                    "last_updated": datetime.now(timezone.utc).isoformat()
                }
            )
            
            return BulkFinancialDataItem(
                ticker=ticker,
                success=True,
                data=response,
                error=None
            )
            
        except Exception as e:
            # Handle individual ticker failure
            error_message = str(e)
            if "No data returned" in error_message:
                error_message = f"No financial data found for ticker {ticker}"
            elif "Invalid ticker" in error_message:
                error_message = f"Invalid or unknown ticker: {ticker}"
            
            return BulkFinancialDataItem(
                ticker=ticker,
                success=False,
                data=None,
                error=error_message
            )
    
    # Process all tickers in parallel with concurrency limit
    semaphore = asyncio.Semaphore(10)  # Limit concurrent operations to avoid overwhelming DB/APIs
    
    async def process_with_limit(ticker: str):
        async with semaphore:
            return await process_ticker(ticker)
    
    # Execute all tickers in parallel (300s endpoint-level timeout)
    tasks = [process_with_limit(ticker) for ticker in request.tickers]
    try:
        results = await asyncio.wait_for(
            asyncio.gather(*tasks, return_exceptions=True),
            timeout=300,
        )
    except asyncio.TimeoutError:
        raise HTTPException(status_code=504, detail="Bulk financial data request timed out after 300s.")
    
    # Count successful and failed results
    successful_count = 0
    failed_count = 0
    
    for i, result in enumerate(results):
        if isinstance(result, Exception):
            # Handle unexpected exceptions
            error_message = f"Unexpected error processing {request.tickers[i]}: {str(result)}"
            results[i] = BulkFinancialDataItem(
                ticker=request.tickers[i],
                success=False,
                data=None,
                error=error_message
            )
            failed_count += 1
        elif result.success:
            successful_count += 1
        else:
            failed_count += 1
    
    return BulkFinancialDataResponse(
        results=results,
        metadata={
            "total_requested": len(request.tickers),
            "successful": successful_count,
            "failed": failed_count,
            "period_type": request.period_type.value,
            "quarters_requested": request.quarters if request.quarters else None,
            "date_range": {
                "start": start_date.isoformat(),
                "end": end_date.isoformat()
            },
            "force_refresh": request.force_refresh,
            "processed_at": datetime.now(timezone.utc).isoformat()
        }
    )