stock-oracle/app/api/v1/endpoints/filings.py

"""
SEC Filings endpoints: search, document listing, and exhibit extraction.
"""

import asyncio
import logging
from datetime import datetime, timezone
from typing import List, Optional

from fastapi import APIRouter, Depends, HTTPException, Query, Response
from sqlalchemy.ext.asyncio import AsyncSession

from app.core.database import get_db, AsyncSessionLocal
from app.schemas.filing import (
    BulkExhibitItem,
    BulkExhibitRequest,
    BulkExhibitResponse,
    BulkFilingSearchItem,
    BulkFilingSearchRequest,
    BulkFilingSearchResponse,
    BulkParseRequest,
    BulkParseResponse,
    ExhibitContentResponse,
    FilingDocumentInfo,
    FilingDocumentListResponse,
    FilingEventResponse,
    FilingEventsSearchResponse,
    FilingSearchResponse,
    FilingSummary,
)
from app.models.filing_event import SECFilingEvent
from app.services.sec_filings_service import sec_filings_service
from app.services.sec_8k_parser import sec_8k_parser
from app.utils.cache import with_cache, build_cache_key, get_negative_cached, set_negative_cached
from sqlalchemy import select, and_, func as sql_func

router = APIRouter()
logger = logging.getLogger("app.api.v1.filings")


@router.get(
    "/search/{ticker}",
    response_model=FilingSearchResponse,
    summary="Search SEC filings for a ticker",
    description=(
        "Search SEC filings for the given ticker. Supported form types: **8-K, 6-K, 20-F, 40-F**.\n\n"
        "Auto-indexes filings from EDGAR on first request (or when `force_refresh=true`). "
        "Results are cached for 1 hour.\n\n"
        "**현재 DB 보유**: 1994-01-05 ~ 현재, 1598 티커. "
        "처음 조회하는 티커는 SEC EDGAR에서 자동 인덱싱 (수 초 소요).\n\n"
        "**Example**: `GET /filings/search/AAPL?form_type=8-K&limit=10`"
    ),
)
@with_cache(namespace="filings:search", ttl=3600, key_params=["ticker", "form_type", "start_date", "end_date", "limit", "offset"])
async def search_filings(
    ticker: str,
    response: Response,
    form_type: Optional[str] = Query(
        None,
        description="Comma-separated form types (e.g. '8-K,6-K'). Default: all supported.",
    ),
    start_date: Optional[str] = Query(None, description="Start date YYYY-MM-DD"),
    end_date: Optional[str] = Query(None, description="End date YYYY-MM-DD"),
    limit: int = Query(20, ge=1, le=100),
    offset: int = Query(0, ge=0),
    force_refresh: bool = Query(False, description="Force re-indexing from SEC"),
    db: AsyncSession = Depends(get_db),
):
    """Search SEC filings for a ticker. Auto-indexes from SEC if not in DB."""
    form_types_set = None
    if form_type:
        form_types_set = {ft.strip().upper() for ft in form_type.split(",") if ft.strip()}
        unsupported = form_types_set - sec_filings_service.SUPPORTED_FORM_TYPES
        if unsupported:
            raise HTTPException(
                status_code=400,
                detail=f"Unsupported form types: {unsupported}. Supported: {sec_filings_service.SUPPORTED_FORM_TYPES}",
            )

    start_dt = None
    end_dt = None
    try:
        if start_date:
            start_dt = datetime.strptime(start_date, "%Y-%m-%d").replace(tzinfo=timezone.utc)
        if end_date:
            end_dt = datetime.strptime(end_date, "%Y-%m-%d").replace(
                hour=23, minute=59, second=59, tzinfo=timezone.utc
            )
    except ValueError:
        raise HTTPException(status_code=400, detail="Invalid date format. Use YYYY-MM-DD")

    if force_refresh:
        try:
            await sec_filings_service.index_filings(
                db, ticker, form_types=form_types_set, force_refresh=True
            )
        except ValueError as e:
            raise HTTPException(status_code=404, detail=str(e))
        except Exception as e:
            logger.error(f"Force refresh indexing failed for {ticker}: {e}")
            raise HTTPException(status_code=502, detail=f"SEC indexing failed: {e}")

    try:
        filings, total_count = await asyncio.wait_for(
            sec_filings_service.search_filings(
                db,
                ticker,
                form_types=form_types_set,
                start_date=start_dt,
                end_date=end_dt,
                limit=limit,
                offset=offset,
            ),
            timeout=120,
        )
    except asyncio.TimeoutError:
        raise HTTPException(status_code=504, detail="Filing search timed out after 120s.")
    except ValueError as e:
        raise HTTPException(status_code=404, detail=str(e))
    except Exception as e:
        logger.error(f"Filing search failed for {ticker}: {e}")
        raise HTTPException(status_code=502, detail=f"Filing search failed: {e}")

    summaries = []
    for f in filings:
        docs_count = len(f.documents_json) if f.documents_json else None
        summaries.append(
            FilingSummary(
                accession_number=f.accession_number,
                form_type=f.form_type,
                filing_date=f.filing_date.strftime("%Y-%m-%d"),
                accepted_at=f.accepted_at.isoformat() if f.accepted_at else None,
                primary_document=f.primary_document,
                filing_description=f.filing_description,
                documents_count=docs_count,
                parsed_status=f.parsed_status,
                items=f.items_json if isinstance(f.items_json, list) else None,
            )
        )

    return FilingSearchResponse(
        ticker=ticker.upper(),
        filings=summaries,
        total_count=total_count,
        metadata={
            "limit": limit,
            "offset": offset,
            "form_types": list(form_types_set) if form_types_set else None,
        },
    )


@router.get(
    "/documents/{accession_number}",
    response_model=FilingDocumentListResponse,
    summary="List documents in a SEC filing",
    description=(
        "List all documents attached to a SEC filing by accession number.\n\n"
        "Returns filename, document type, size, and SEC URL for each document. "
        "Cached for 24 hours.\n\n"
        "**Example**: `GET /filings/documents/0001193125-24-123456`"
    ),
)
@with_cache(namespace="filings:documents", ttl=86400, key_params=["accession_number"])
async def get_filing_documents(
    accession_number: str,
    response: Response,
    db: AsyncSession = Depends(get_db),
):
    """List all documents within a SEC filing."""
    try:
        documents = await asyncio.wait_for(
            sec_filings_service.get_filing_documents(db, accession_number),
            timeout=30,
        )
    except asyncio.TimeoutError:
        raise HTTPException(status_code=504, detail="Document listing timed out after 30s.")
    except ValueError as e:
        raise HTTPException(status_code=404, detail=str(e))
    except Exception as e:
        logger.error(f"Document listing failed for {accession_number}: {e}")
        raise HTTPException(status_code=502, detail=f"Document listing failed: {e}")

    doc_infos = [FilingDocumentInfo(**d) for d in documents]
    return FilingDocumentListResponse(
        accession_number=accession_number,
        documents=doc_infos,
        metadata={"total_documents": len(doc_infos)},
    )


@router.get(
    "/exhibit/{accession_number}",
    response_model=ExhibitContentResponse,
    summary="Extract exhibit content from a filing",
    description=(
        "Extract the text content of a specific exhibit (e.g., press release **EX-99.1**) "
        "from a SEC filing.\n\n"
        "Returns the full text content along with content type, filename, and SEC URL. "
        "404 responses are negative-cached for 1 hour. Cached for 24 hours.\n\n"
        "**Example**: `GET /filings/exhibit/0001193125-24-123456?exhibit_type=EX-99.1`"
    ),
)
@with_cache(namespace="filings:exhibit", ttl=86400, key_params=["accession_number", "exhibit_type"])
async def get_exhibit_content(
    accession_number: str,
    response: Response,
    exhibit_type: str = Query("EX-99.1", description="Exhibit type (e.g. EX-99.1)"),
    db: AsyncSession = Depends(get_db),
):
    """Extract exhibit content (e.g., press release EX-99.1) from a filing."""
    # Fast path: check negative cache (exhibit known not to exist)
    neg_key = build_cache_key("filings:exhibit:404", accession_number, exhibit_type)
    if await get_negative_cached(neg_key):
        raise HTTPException(
            status_code=404,
            detail=f"Exhibit {exhibit_type} not found in filing {accession_number}",
        )

    try:
        result = await asyncio.wait_for(
            sec_filings_service.get_exhibit_content(db, accession_number, exhibit_type),
            timeout=25,
        )
    except asyncio.TimeoutError:
        raise HTTPException(status_code=504, detail="Exhibit extraction timed out after 25s.")
    except ValueError as e:
        # Cache this "not found" result so subsequent requests skip SEC entirely
        await set_negative_cached(neg_key, ttl=3600)
        raise HTTPException(status_code=404, detail=str(e))
    except Exception as e:
        logger.error(f"Exhibit extraction failed for {accession_number}/{exhibit_type}: {e}")
        raise HTTPException(status_code=502, detail=f"Exhibit extraction failed: {e}")

    return ExhibitContentResponse(
        accession_number=accession_number,
        exhibit_type=exhibit_type,
        content=result["content"],
        content_type=result.get("content_type"),
        filename=result.get("filename"),
        url=result.get("url"),
    )


@router.post(
    "/search/bulk",
    response_model=BulkFilingSearchResponse,
    summary="Bulk search SEC filings for multiple tickers",
    description=(
        "Search SEC filings for up to many tickers in a single request. "
        "Auto-indexes from EDGAR for any ticker not yet in the database.\n\n"
        "**Timeout**: 600 seconds. Each ticker is processed concurrently.\n\n"
        "**Example body**:\n"
        "```json\n"
        '{"tickers": ["AAPL", "MSFT", "NVDA"], "form_type": "8-K", '
        '"start_date": "2024-01-01", "limit_per_ticker": 5}\n'
        "```"
    ),
)
@with_cache(namespace="filings:search:bulk", ttl=3600, key_params=["request"])
async def search_filings_bulk(
    request: BulkFilingSearchRequest,
    response: Response,
    db: AsyncSession = Depends(get_db),
):
    """Bulk search SEC filings for multiple tickers. Auto-indexes from SEC if not in DB."""
    form_types_set = None
    if request.form_type:
        form_types_set = {ft.strip().upper() for ft in request.form_type.split(",") if ft.strip()}
        unsupported = form_types_set - sec_filings_service.SUPPORTED_FORM_TYPES
        if unsupported:
            raise HTTPException(
                status_code=400,
                detail=f"Unsupported form types: {unsupported}.",
            )

    start_dt = None
    end_dt = None
    try:
        if request.start_date:
            start_dt = datetime.strptime(request.start_date, "%Y-%m-%d").replace(tzinfo=timezone.utc)
        if request.end_date:
            end_dt = datetime.strptime(request.end_date, "%Y-%m-%d").replace(
                hour=23, minute=59, second=59, tzinfo=timezone.utc
            )
    except ValueError:
        raise HTTPException(status_code=400, detail="Invalid date format. Use YYYY-MM-DD")

    try:
        raw_results, elapsed = await asyncio.wait_for(
            sec_filings_service.search_filings_bulk(
                db,
                request.tickers,
                form_types=form_types_set,
                start_date=start_dt,
                end_date=end_dt,
                limit_per_ticker=request.limit_per_ticker,
            ),
            timeout=600,
        )
    except asyncio.TimeoutError:
        raise HTTPException(status_code=504, detail="Bulk filing search timed out after 600s.")
    except Exception as e:
        logger.error(f"Bulk filing search failed: {e}")
        raise HTTPException(status_code=502, detail=f"Bulk filing search failed: {e}")

    items = []
    successful = 0
    failed = 0
    for r in raw_results:
        summaries = []
        for f in r["filings"]:
            docs_count = len(f.documents_json) if f.documents_json else None
            summaries.append(
                FilingSummary(
                    accession_number=f.accession_number,
                    form_type=f.form_type,
                    filing_date=f.filing_date.strftime("%Y-%m-%d"),
                    accepted_at=f.accepted_at.isoformat() if f.accepted_at else None,
                    primary_document=f.primary_document,
                    filing_description=f.filing_description,
                    documents_count=docs_count,
                )
            )
        items.append(BulkFilingSearchItem(
            ticker=r["ticker"],
            success=r["success"],
            filings=summaries,
            total_count=r["total_count"],
            error=r["error"],
        ))
        if r["success"]:
            successful += 1
        else:
            failed += 1

    return BulkFilingSearchResponse(
        results=items,
        total_tickers=len(items),
        successful_count=successful,
        failed_count=failed,
        query_time_seconds=round(elapsed, 3),
        metadata={"form_types": list(form_types_set) if form_types_set else None},
    )


@router.post(
    "/exhibit/bulk",
    response_model=BulkExhibitResponse,
    summary="Bulk fetch exhibit content",
    description=(
        "Fetch exhibit content for multiple accession numbers in one request. "
        "Up to 4 concurrent fetches; max 300 second timeout.\n\n"
        "**Example body**:\n"
        "```json\n"
        '{"items": [{"accession_number": "0001193125-24-123456", "exhibit_type": "EX-99.1"}]}\n'
        "```"
    ),
)
async def get_exhibit_bulk(
    request: BulkExhibitRequest,
    db: AsyncSession = Depends(get_db),
):
    """Bulk fetch exhibit content for multiple accession numbers."""
    import time as _time
    t0 = _time.monotonic()

    async def _fetch_one(item: dict) -> BulkExhibitItem:
        accession_number = item.get("accession_number", "")
        exhibit_type = item.get("exhibit_type", "EX-99.1")
        neg_key = build_cache_key("filings:exhibit:404", accession_number, exhibit_type)
        if await get_negative_cached(neg_key):
            return BulkExhibitItem(
                accession_number=accession_number,
                exhibit_type=exhibit_type,
                success=False,
                error=f"Exhibit {exhibit_type} not found in filing {accession_number}",
            )
        # Each concurrent call gets its own DB session to avoid session contention
        async with AsyncSessionLocal() as session:
            try:
                result = await asyncio.wait_for(
                    sec_filings_service.get_exhibit_content(session, accession_number, exhibit_type),
                    timeout=25,
                )
                return BulkExhibitItem(
                    accession_number=accession_number,
                    exhibit_type=exhibit_type,
                    success=True,
                    content=result["content"],
                    content_type=result.get("content_type"),
                    filename=result.get("filename"),
                    url=result.get("url"),
                )
            except ValueError as e:
                await set_negative_cached(neg_key, ttl=3600)
                return BulkExhibitItem(
                    accession_number=accession_number,
                    exhibit_type=exhibit_type,
                    success=False,
                    error=str(e),
                )
            except Exception as e:
                return BulkExhibitItem(
                    accession_number=accession_number,
                    exhibit_type=exhibit_type,
                    success=False,
                    error=str(e),
                )

    sem = asyncio.Semaphore(4)

    async def _fetch_one_limited(item):
        async with sem:
            return await _fetch_one(item)

    raw = await asyncio.wait_for(
        asyncio.gather(*[_fetch_one_limited(item) for item in request.items], return_exceptions=True),
        timeout=300,
    )
    results: List[BulkExhibitItem] = []
    for r in raw:
        if isinstance(r, Exception):
            results.append(BulkExhibitItem(
                accession_number="unknown",
                exhibit_type="unknown",
                success=False,
                error=str(r),
            ))
        else:
            results.append(r)

    successful = sum(1 for r in results if r.success)
    elapsed = _time.monotonic() - t0

    return BulkExhibitResponse(
        results=results,
        total_items=len(results),
        successful_count=successful,
        failed_count=len(results) - successful,
        query_time_seconds=round(elapsed, 3),
    )


# ── Filing Events ────────────────────────────────────────────────────────────

@router.get(
    "/events/{ticker}",
    response_model=FilingEventsSearchResponse,
    summary="Get parsed 8-K events for a ticker",
    description=(
        "Returns structured events parsed from 8-K filings. Each event corresponds to one "
        "8-K Item (e.g., Item 8.01 → other_material_event, Item 2.02 → earnings_result).\\n\\n"
        "If there are unprocessed (pending) filings, they are lazily parsed on first request.\\n\\n"
        "**Example**: `GET /filings/events/AVGO?start_date=2026-04-01&event_type=other_material_event`"
    ),
)
@with_cache(namespace="filings:events", ttl=1800, key_params=["ticker", "start_date", "end_date", "event_type", "limit", "offset"])
async def get_filing_events(
    ticker: str,
    response: Response,
    start_date: Optional[str] = Query(None, description="Start date YYYY-MM-DD"),
    end_date: Optional[str] = Query(None, description="End date YYYY-MM-DD"),
    event_type: Optional[str] = Query(None, description="Filter by event type (e.g. other_material_event)"),
    limit: int = Query(20, ge=1, le=200),
    offset: int = Query(0, ge=0),
    db: AsyncSession = Depends(get_db),
):
    """Return parsed 8-K events for a ticker. Lazy-parses any pending filings first."""
    ticker_upper = ticker.upper()

    start_dt = end_dt = None
    try:
        if start_date:
            start_dt = datetime.strptime(start_date, "%Y-%m-%d").replace(tzinfo=timezone.utc)
        if end_date:
            end_dt = datetime.strptime(end_date, "%Y-%m-%d").replace(
                hour=23, minute=59, second=59, tzinfo=timezone.utc
            )
    except ValueError:
        raise HTTPException(status_code=400, detail="Invalid date format. Use YYYY-MM-DD")

    # Lazy-parse any pending 8-Ks for this ticker (limit 20 to avoid long waits)
    try:
        await asyncio.wait_for(
            sec_8k_parser.parse_bulk(db, tickers=[ticker_upper], limit=20),
            timeout=60,
        )
    except (asyncio.TimeoutError, Exception) as e:
        logger.warning(f"Lazy parse timed out/failed for {ticker_upper}: {e}")

    # Query events
    conditions = [SECFilingEvent.ticker == ticker_upper]
    if start_dt:
        conditions.append(SECFilingEvent.filing_date >= start_dt)
    if end_dt:
        conditions.append(SECFilingEvent.filing_date <= end_dt)
    if event_type:
        conditions.append(SECFilingEvent.event_type == event_type)

    where_clause = and_(*conditions)

    count_result = await db.execute(
        select(sql_func.count()).select_from(SECFilingEvent).where(where_clause)
    )
    total_count = count_result.scalar() or 0

    rows_result = await db.execute(
        select(SECFilingEvent)
        .where(where_clause)
        .order_by(SECFilingEvent.filing_date.desc())
        .limit(limit)
        .offset(offset)
    )
    rows = rows_result.scalars().all()

    events = [
        FilingEventResponse(
            id=str(row.id),
            ticker=row.ticker,
            accession_number=row.accession_number,
            form_type=row.form_type,
            filing_date=row.filing_date.strftime("%Y-%m-%d"),
            item_number=row.item_number,
            event_type=row.event_type,
            title=row.title,
            summary=row.summary,
            content_source=row.content_source,
        )
        for row in rows
    ]

    return FilingEventsSearchResponse(
        ticker=ticker_upper,
        events=events,
        total_count=total_count,
        metadata={
            "limit": limit,
            "offset": offset,
            "event_type": event_type,
        },
    )


@router.post(
    "/events/parse/bulk",
    response_model=BulkParseResponse,
    summary="Bulk parse pending 8-K filings",
    description=(
        "Parse pending 8-K filings and create structured events. "
        "Use this to backfill events for existing DB records.\\n\\n"
        "**Example body**: `{\"tickers\": [\"AVGO\", \"AAPL\"], \"limit\": 50}`\\n"
        "Omit `tickers` to parse all pending filings (up to `limit`)."
    ),
)
async def parse_8k_bulk(
    request: BulkParseRequest,
    db: AsyncSession = Depends(get_db),
):
    """Bulk parse pending 8-K filings and persist events."""
    try:
        result = await asyncio.wait_for(
            sec_8k_parser.parse_bulk(db, tickers=request.tickers, limit=request.limit),
            timeout=600,
        )
    except asyncio.TimeoutError:
        raise HTTPException(status_code=504, detail="Bulk parse timed out after 600s")
    except Exception as e:
        logger.error(f"Bulk parse failed: {e}")
        raise HTTPException(status_code=502, detail=f"Bulk parse failed: {e}")

    return BulkParseResponse(
        succeeded=result["succeeded"],
        failed=result["failed"],
        skipped=result["skipped"],
        total=result["total"],
        query_time_seconds=result["elapsed"],
    )