stock-oracle/tests/test_sec_edgar_15year.py

"""
Comprehensive test for 15-year SEC EDGAR financial data fetching
Tests the new SEC-only implementation without yfinance_plus via API
"""

import requests
import asyncio
from datetime import datetime, timezone, timedelta
import time
import json


class TestSECEdgar15Year:
    """Test 15-year SEC EDGAR financial data fetching via API"""
    
    def setup_method(self):
        """Setup test environment"""
        self.api_url = "http://localhost:18001/api/v1"
        
    def test_sec_only_financial_data_aapl(self):
        """Test that AAPL financial data comes from SEC only"""
        print(f"\n=== Testing SEC EDGAR 15-year data for AAPL ===")
        
        # Test 15-year range
        end_date = datetime.now()
        start_date = end_date - timedelta(days=15*365)
        
        print(f"Date range: {start_date.date()} to {end_date.date()}")
        
        request_data = {
            "ticker": "AAPL",
            "start_date": start_date.strftime('%Y-%m-%d'),
            "end_date": end_date.strftime('%Y-%m-%d'),
            "period_type": "all",
            "include_metrics": True,
            "force_refresh": True
        }
        
        try:
            print("Requesting financial data...")
            start_time = time.time()
            
            response = requests.post(
                f"{self.api_url}/financial/data",
                json=request_data,
                timeout=120
            )
            
            end_time = time.time()
            duration = end_time - start_time
            
            assert response.status_code == 200, f"API request failed: {response.status_code}"
            
            data = response.json()
            financial_data = data.get('financial_data', [])
            print(f"Found {len(financial_data)} financial records in {duration:.1f}s")
            
            # Verify data source is SEC
            sec_records = [
                record for record in financial_data 
                if record.get('data_source') == 'SEC_EDGAR'
            ]
            print(f"SEC EDGAR records: {len(sec_records)}")
            
            # Verify we have historical data (should go back multiple years)
            if financial_data:
                dates = [datetime.fromisoformat(record['period_date'].replace('Z', '+00:00')) for record in financial_data]
                dates.sort()
                oldest_date = dates[0]
                newest_date = dates[-1]
                
                print(f"Date range in data: {oldest_date.date()} to {newest_date.date()}")
                
                years_span = (newest_date - oldest_date).days / 365.25
                print(f"Years span: {years_span:.1f} years")
                
                # Should have at least 3 years of data
                assert years_span >= 3, f"Expected at least 3 years, got {years_span:.1f}"
                
                # Verify data quality
                for i, record in enumerate(financial_data[:3]):
                    period_date = record['period_date']
                    print(f"\nRecord {i+1} for {period_date[:10]}:")
                    print(f"  Revenue: ${record.get('revenue', 0):,.0f}")
                    print(f"  Net Income: ${record.get('net_income', 0):,.0f}")
                    print(f"  Total Assets: ${record.get('total_assets', 0):,.0f}")
                    print(f"  Data Source: {record.get('data_source')}")
                    print(f"  Is Estimated: {record.get('is_estimated')}")
                    
                    # Verify it's real SEC data
                    assert record.get('data_source') == 'SEC_EDGAR'
                    assert record.get('is_estimated') == False
            
            assert len(financial_data) > 0, "Should have financial data from SEC"
            print("✅ Test passed: SEC EDGAR data retrieved successfully")
            
        except Exception as e:
            print(f"❌ Test failed: {e}")
            raise
            
    def test_multiple_tickers_sec_data(self):
        """Test SEC data fetching for multiple tickers"""
        print(f"\n=== Testing Multiple Tickers SEC Data ===")
        
        tickers = ["AAPL", "MSFT"]
        end_date = datetime.now()
        start_date = end_date - timedelta(days=5*365)  # 5 years
        
        for ticker in tickers:
            print(f"\nTesting {ticker}...")
            
            request_data = {
                "ticker": ticker,
                "start_date": start_date.strftime('%Y-%m-%d'),
                "end_date": end_date.strftime('%Y-%m-%d'),
                "period_type": "all",
                "include_metrics": True,
                "force_refresh": False
            }
            
            try:
                response = requests.post(
                    f"{self.api_url}/financial/data",
                    json=request_data,
                    timeout=60
                )
                
                if response.status_code == 200:
                    data = response.json()
                    financial_data = data.get('financial_data', [])
                    print(f"  Records found: {len(financial_data)}")
                    
                    if financial_data:
                        # Show latest record
                        latest = max(financial_data, key=lambda x: x['period_date'])
                        print(f"  Latest period: {latest['period_date'][:10]}")
                        print(f"  Revenue: ${latest.get('revenue', 0):,.0f}")
                        print(f"  Data source: {latest.get('data_source')}")
                        
                        # Verify all are SEC data
                        sec_count = sum(1 for r in financial_data if r.get('data_source') == 'SEC_EDGAR')
                        print(f"  SEC records: {sec_count}/{len(financial_data)}")
                        
                        assert sec_count == len(financial_data), f"Expected all SEC data, got {sec_count}/{len(financial_data)}"
                else:
                    print(f"  ❌ Failed: {response.status_code}")
                    
            except Exception as e:
                print(f"  ❌ Error: {e}")
                    
    def test_no_yfinance_plus_dependency(self):
        """Verify yfinance_plus is not used for financial data"""
        print(f"\n=== Testing No yfinance_plus Dependency ===")
        
        # Test that yfinance_plus is not used
        try:
            import yfinance_plus
            print("WARNING: yfinance_plus is still available in environment")
        except ImportError:
            print("✓ yfinance_plus correctly not available")
        
        # Test financial data request to ensure SEC is used
        end_date = datetime.now()
        start_date = end_date - timedelta(days=365)
        
        request_data = {
            "ticker": "AAPL",
            "start_date": start_date.strftime('%Y-%m-%d'),
            "end_date": end_date.strftime('%Y-%m-%d'),
            "period_type": "quarterly",
            "include_metrics": True,
            "force_refresh": False
        }
        
        try:
            response = requests.post(
                f"{self.api_url}/financial/data",
                json=request_data,
                timeout=60
            )
            
            assert response.status_code == 200, f"API request failed: {response.status_code}"
            
            data = response.json()
            financial_data = data.get('financial_data', [])
            
            # All financial data should be from SEC
            for record in financial_data:
                assert record.get('data_source') == 'SEC_EDGAR', f"Expected SEC_EDGAR, got {record.get('data_source')}"
                assert record.get('is_estimated') == False, "Should not be estimated data"
            
            print(f"✓ All {len(financial_data)} financial records are from SEC EDGAR")
            
        except Exception as e:
            print(f"❌ Error testing financial data: {e}")
            raise
    
    def test_price_data_separation(self):
        """Test that price data still works with yfinance"""
        print(f"\n=== Testing Price Data Separation ===")
        
        end_date = datetime.now()
        start_date = end_date - timedelta(days=30)  # 30 days
        
        request_data = {
            "ticker": "AAPL",
            "start_date": start_date.strftime('%Y-%m-%d'),
            "end_date": end_date.strftime('%Y-%m-%d'),
            "interval": "1d"
        }
        
        try:
            response = requests.post(
                f"{self.api_url}/financial/price-data",
                json=request_data,
                timeout=60
            )
            
            if response.status_code == 200:
                data = response.json()
                price_data = data.get('price_data', [])
                print(f"Price records found: {len(price_data)}")
                
                if price_data:
                    latest_price = max(price_data, key=lambda x: x['date'])
                    print(f"Latest price date: {latest_price['date'][:10]}")
                    print(f"Latest close: ${latest_price.get('close', 0):.2f}")
                    print(f"Price data source: {latest_price.get('data_source')}")
                    
                    # Price data should be from Yahoo Finance
                    yahoo_count = sum(1 for p in price_data if p.get('data_source') == 'YAHOO_FINANCE')
                    print(f"Yahoo Finance price records: {yahoo_count}/{len(price_data)}")
                    
                    assert yahoo_count > 0, "Should have price data from Yahoo Finance"
            else:
                print(f"Price data request failed: {response.status_code}")
                
        except Exception as e:
            print(f"❌ Error testing price data: {e}")


if __name__ == "__main__":
    def run_tests():
        test_instance = TestSECEdgar15Year()
        test_instance.setup_method()
        
        try:
            test_instance.test_sec_only_financial_data_aapl()
            test_instance.test_multiple_tickers_sec_data()
            test_instance.test_no_yfinance_plus_dependency()
            test_instance.test_price_data_separation()
            print("\n✅ All tests completed successfully!")
        except Exception as e:
            print(f"\n❌ Test failed: {e}")
            raise
    
    # Run the tests
    run_tests()