diff --git a/app/api/v1/endpoints/attention.py b/app/api/v1/endpoints/attention.py index f44ec50..68cd109 100644 --- a/app/api/v1/endpoints/attention.py +++ b/app/api/v1/endpoints/attention.py @@ -228,11 +228,14 @@ async def admin_collect_gdelt( "/entity/{ticker}", response_model=EntityResolveResponse, summary="Get entity mapping for a ticker", - description=( - "Returns the stored entity mapping for a ticker: canonical name, Wikipedia title, " - "GDELT query string, and resolver confidence score.\n\n" - "Returns 404 if no mapping exists — run `POST /admin/resolve/{ticker}` first." - ), + description=""" + Returns the stored entity mapping for a ticker: canonical name, Wikipedia title, + GDELT query string, and resolver confidence score. + + Returns **404** if no mapping exists — run `POST /admin/resolve/{ticker}` first. + + **Example**: `GET /attention/entity/AAPL` + """, ) async def get_entity( ticker: str, @@ -265,18 +268,34 @@ async def get_entity( "/event/{ticker}", response_model=EventAttentionResponse, summary="Get attention features for a ticker on an event date", - description=( - "Returns Wikipedia pageview spike/z-score and GDELT news article counts " - "for the given ticker centered on `event_date`.\n\n" - "**Wikipedia data** is collected on-demand if not in DB. " - "**GDELT data** is never collected on-demand — it must be pre-populated via " - "`POST /admin/collect/gdelt/{ticker}` (scheduler). " - "Check `news.gdelt_status` in the response to understand data availability:\n\n" - "- `collected` — GDELT was collected; counts are accurate (0 means genuinely no articles)\n" - "- `not_collected` — scheduler has not run for this date yet\n" - "- `not_available` — event date is before GDELT V2 coverage (2017-01-01)\n\n" - "If no entity mapping exists, resolution runs automatically before collection." - ), + description=""" + Returns Wikipedia pageview spike/z-score and GDELT news volume for a ticker + centered on a specific event date. Designed for event-driven backtesting. + + **Wikipedia signals** (collected on-demand): + - `wiki.views` — raw pageview count on `event_date` + - `wiki.spike_10d` — views / 10-day median baseline; >1 = above-average interest + - `wiki.zscore_20d` — standard-deviation units above 20-day mean + + **GDELT news signals** (pre-populated by scheduler only): + - `news.article_count_1d` — articles published on `event_date` + - `news.article_count_3d` — articles in `event_date ± 1 day` window + - `news.unique_domains_3d` — distinct publisher domains in that window + - `news.gdelt_status` — data availability flag: + - `collected` — scheduler ran; counts are accurate (0 = genuinely no articles) + - `not_collected` — scheduler has not run yet; use `POST /admin/collect/gdelt/{ticker}` + - `not_available` — event date is before GDELT V2 coverage (2017-01-01) + + **Auto-resolution**: if no entity mapping exists, resolution runs automatically first. + + **Examples**: + - `GET /attention/event/AAPL?event_date=2024-02-01` — Q1 earnings day attention + - `GET /attention/event/NVDA?event_date=2024-05-22` — post-earnings spike + """, + responses={ + 404: {"description": "Ticker not found or entity resolution failed"}, + 500: {"description": "Feature materialization or collection error"}, + }, ) async def get_event_attention( ticker: str, diff --git a/app/api/v1/endpoints/database.py b/app/api/v1/endpoints/database.py index 36b2239..0bb08f2 100644 --- a/app/api/v1/endpoints/database.py +++ b/app/api/v1/endpoints/database.py @@ -17,7 +17,18 @@ from app.schemas.financial import DataSource router = APIRouter() logger = logging.getLogger(__name__) -@router.get("/stats") +@router.get( + "/stats", + summary="Database record counts and date ranges", + description=""" + Returns aggregate statistics across all core tables: + + - `companies` — total companies, how many have financial/price data + - `financial_data` — total records, real vs estimated, date range, breakdown by source + - `price_data` — total records, date range, list of tickers + - `calculated_metrics` — total records and date range + """, +) async def get_database_stats(db: AsyncSession = Depends(get_db)) -> Dict[str, Any]: """ 데이터베이스 통계 정보를 반환합니다. @@ -156,7 +167,7 @@ async def get_database_stats(db: AsyncSession = Depends(get_db)) -> Dict[str, An detail=f"Failed to fetch database statistics: {str(e)}" ) -@router.get("/health") +@router.get("/health", summary="Database connection health check") async def get_database_health(db: AsyncSession = Depends(get_db)) -> Dict[str, Any]: """ 데이터베이스 연결 상태를 확인합니다. @@ -178,7 +189,7 @@ async def get_database_health(db: AsyncSession = Depends(get_db)) -> Dict[str, A detail=f"Database connection failed: {str(e)}" ) -@router.get("/tables") +@router.get("/tables", summary="Table row counts for all core tables") async def get_table_info(db: AsyncSession = Depends(get_db)) -> Dict[str, Any]: """ 데이터베이스 테이블 정보를 반환합니다. @@ -213,7 +224,7 @@ async def get_table_info(db: AsyncSession = Depends(get_db)) -> Dict[str, Any]: detail=f"Failed to fetch table info: {str(e)}" ) -@router.post("/cleanup/duplicates") +@router.post("/cleanup/duplicates", summary="Remove duplicate financial and metrics records") async def cleanup_duplicate_records(db: AsyncSession = Depends(get_db)) -> Dict[str, Any]: """ Remove duplicate financial and metrics records, keeping the most recent real data. @@ -263,7 +274,7 @@ async def cleanup_duplicate_records(db: AsyncSession = Depends(get_db)) -> Dict[ detail=f"Failed to cleanup duplicates: {str(e)}" ) -@router.get("/tickers") +@router.get("/tickers", summary="List tickers available in the database") async def get_available_tickers(db: AsyncSession = Depends(get_db)) -> Dict[str, List[str]]: """ 사용 가능한 종목 목록을 반환합니다. @@ -306,7 +317,18 @@ async def get_available_tickers(db: AsyncSession = Depends(get_db)) -> Dict[str, # ETF persisted data browsing # ========================== -@router.get("/etf/snapshots") +@router.get( + "/etf/snapshots", + summary="List persisted ETF holdings snapshots", + description=""" + Browse ETF holdings snapshots stored in the database. Each snapshot represents + the portfolio as reported in a SEC 13-F filing. + + Filter by `ticker`, `start_date`, `end_date`. Results are ordered by snapshot date (newest first). + + **Example**: `GET /database/etf/snapshots?ticker=SPY&limit=10` + """, +) async def list_etf_snapshots( ticker: str | None = None, start_date: str | None = None, @@ -366,7 +388,7 @@ async def list_etf_snapshots( raise HTTPException(status_code=500, detail="Failed to list ETF snapshots") -@router.get("/etf/snapshot/{snapshot_id}") +@router.get("/etf/snapshot/{snapshot_id}", summary="Get ETF snapshot with full holdings list") async def get_etf_snapshot(snapshot_id: str, db: AsyncSession = Depends(get_db)): try: from uuid import UUID @@ -411,7 +433,18 @@ async def get_etf_snapshot(snapshot_id: str, db: AsyncSession = Depends(get_db)) # Financial records browsing list # ============================== -@router.get("/financial/records") +@router.get( + "/financial/records", + summary="Browse raw financial data records", + description=""" + List raw financial data rows from the `financial_data` table. + + Supports filtering by `ticker`, `period_type` (`quarterly`/`annual`), + `start_date`, and `end_date`. Results ordered by `period_date` descending. + + **Example**: `GET /database/financial/records?ticker=AAPL&period_type=quarterly&limit=8` + """, +) async def list_financial_records( ticker: str | None = None, period_type: str | None = None, diff --git a/app/api/v1/endpoints/etf.py b/app/api/v1/endpoints/etf.py index c68ec4d..58f6bc7 100644 --- a/app/api/v1/endpoints/etf.py +++ b/app/api/v1/endpoints/etf.py @@ -33,7 +33,22 @@ class ETFHoldingsOut(BaseModel): error: Optional[str] = None -@router.get("/holdings/{ticker}", response_model=ETFHoldingsOut) +@router.get( + "/holdings/{ticker}", + response_model=ETFHoldingsOut, + summary="Get ETF portfolio holdings", + description=( + "Fetch the constituent holdings of an ETF (e.g., SPY, QQQ, IWM). " + "Data is sourced from SEC 13-F filings and cached for 1 hour.\n\n" + "Use `top_n` to limit to the N largest positions, or `top_percentage` to return " + "the minimal set of holdings that covers X% of the portfolio (e.g., `top_percentage=0.8` " + "for the holdings making up 80% of the ETF).\n\n" + "**Examples**:\n" + "- `GET /etf/holdings/SPY` — all holdings\n" + "- `GET /etf/holdings/QQQ?top_n=10` — top 10 positions\n" + "- `GET /etf/holdings/IWM?top_percentage=0.5` — holdings covering 50% of portfolio" + ), +) @with_cache(namespace="etf:holdings", ttl=3600, key_params=["ticker", "as_of_date", "top_n", "top_percentage"]) async def get_etf_holdings( ticker: str, @@ -90,7 +105,16 @@ class RefreshMapsOut(BaseModel): etf_rows: int = Field(...) -@router.post("/admin/refresh-maps", response_model=RefreshMapsOut) +@router.post( + "/admin/refresh-maps", + response_model=RefreshMapsOut, + summary="Refresh ETF CIK and CUSIP mapping tables", + description=( + "Re-fetches and upserts the ETF→CIK and CUSIP→ticker mapping tables from SEC data. " + "Run this when new ETFs need to be supported. Returns the number of rows updated." + ), + tags=["etf"], +) async def refresh_etf_maps(db: AsyncSession = Depends(get_db)): refreshed = await etf_loader_service.refresh_all(db) return RefreshMapsOut(**refreshed) diff --git a/app/api/v1/endpoints/filings.py b/app/api/v1/endpoints/filings.py index 30abae9..2379ef9 100644 --- a/app/api/v1/endpoints/filings.py +++ b/app/api/v1/endpoints/filings.py @@ -31,7 +31,17 @@ router = APIRouter() logger = logging.getLogger("app.api.v1.filings") -@router.get("/search/{ticker}", response_model=FilingSearchResponse) +@router.get( + "/search/{ticker}", + response_model=FilingSearchResponse, + summary="Search SEC filings for a ticker", + description=( + "Search SEC filings for the given ticker. Supported form types: **8-K, 6-K, 20-F, 40-F**.\n\n" + "Auto-indexes filings from EDGAR on first request (or when `force_refresh=true`). " + "Results are cached for 1 hour.\n\n" + "**Example**: `GET /filings/search/AAPL?form_type=8-K&limit=10`" + ), +) @with_cache(namespace="filings:search", ttl=3600, key_params=["ticker", "form_type", "start_date", "end_date", "limit", "offset"]) async def search_filings( ticker: str, @@ -129,7 +139,17 @@ async def search_filings( ) -@router.get("/documents/{accession_number}", response_model=FilingDocumentListResponse) +@router.get( + "/documents/{accession_number}", + response_model=FilingDocumentListResponse, + summary="List documents in a SEC filing", + description=( + "List all documents attached to a SEC filing by accession number.\n\n" + "Returns filename, document type, size, and SEC URL for each document. " + "Cached for 24 hours.\n\n" + "**Example**: `GET /filings/documents/0001193125-24-123456`" + ), +) @with_cache(namespace="filings:documents", ttl=86400, key_params=["accession_number"]) async def get_filing_documents( accession_number: str, @@ -158,7 +178,18 @@ async def get_filing_documents( ) -@router.get("/exhibit/{accession_number}", response_model=ExhibitContentResponse) +@router.get( + "/exhibit/{accession_number}", + response_model=ExhibitContentResponse, + summary="Extract exhibit content from a filing", + description=( + "Extract the text content of a specific exhibit (e.g., press release **EX-99.1**) " + "from a SEC filing.\n\n" + "Returns the full text content along with content type, filename, and SEC URL. " + "404 responses are negative-cached for 1 hour. Cached for 24 hours.\n\n" + "**Example**: `GET /filings/exhibit/0001193125-24-123456?exhibit_type=EX-99.1`" + ), +) @with_cache(namespace="filings:exhibit", ttl=86400, key_params=["accession_number", "exhibit_type"]) async def get_exhibit_content( accession_number: str, @@ -200,7 +231,21 @@ async def get_exhibit_content( ) -@router.post("/search/bulk", response_model=BulkFilingSearchResponse) +@router.post( + "/search/bulk", + response_model=BulkFilingSearchResponse, + summary="Bulk search SEC filings for multiple tickers", + description=( + "Search SEC filings for up to many tickers in a single request. " + "Auto-indexes from EDGAR for any ticker not yet in the database.\n\n" + "**Timeout**: 600 seconds. Each ticker is processed concurrently.\n\n" + "**Example body**:\n" + "```json\n" + '{"tickers": ["AAPL", "MSFT", "NVDA"], "form_type": "8-K", ' + '"start_date": "2024-01-01", "limit_per_ticker": 5}\n' + "```" + ), +) @with_cache(namespace="filings:search:bulk", ttl=3600, key_params=["request"]) async def search_filings_bulk( request: BulkFilingSearchRequest, @@ -288,7 +333,19 @@ async def search_filings_bulk( ) -@router.post("/exhibit/bulk", response_model=BulkExhibitResponse) +@router.post( + "/exhibit/bulk", + response_model=BulkExhibitResponse, + summary="Bulk fetch exhibit content", + description=( + "Fetch exhibit content for multiple accession numbers in one request. " + "Up to 4 concurrent fetches; max 300 second timeout.\n\n" + "**Example body**:\n" + "```json\n" + '{"items": [{"accession_number": "0001193125-24-123456", "exhibit_type": "EX-99.1"}]}\n' + "```" + ), +) async def get_exhibit_bulk( request: BulkExhibitRequest, db: AsyncSession = Depends(get_db), diff --git a/app/api/v1/endpoints/fred.py b/app/api/v1/endpoints/fred.py index fcee92f..9458829 100644 --- a/app/api/v1/endpoints/fred.py +++ b/app/api/v1/endpoints/fred.py @@ -20,7 +20,7 @@ logger = logging.getLogger("app.api.v1.fred") # Use /proxy/{endpoint} instead for all FRED API access -@router.get("/stats/usage") +@router.get("/stats/usage", summary="FRED API usage statistics and cache performance") async def get_fred_usage_stats( days: int = Query(7, ge=1, le=30, description="Number of days to include in stats"), use_proxy_stats: bool = Query(True, description="Use enhanced proxy service statistics"), @@ -124,7 +124,7 @@ async def get_fred_usage_stats( # Removed: /search endpoint - use /proxy/series/search instead -@router.get("/proxy/{endpoint:path}") +@router.get("/proxy/{endpoint:path}", summary="Universal FRED API proxy") async def fred_proxy_endpoint( endpoint: str, db: AsyncSession = Depends(get_db), @@ -305,7 +305,7 @@ async def fred_proxy_endpoint( ) -@router.get("/endpoints") +@router.get("/endpoints", summary="List supported FRED API endpoints") async def get_supported_fred_endpoints(): """ Get list of supported FRED API endpoints diff --git a/app/api/v1/endpoints/news.py b/app/api/v1/endpoints/news.py index 0575ef7..ca2a6b8 100644 --- a/app/api/v1/endpoints/news.py +++ b/app/api/v1/endpoints/news.py @@ -75,7 +75,24 @@ class NewsSocialResponse(BaseModel): summary: NewsSocialSummarySchema -@router.get("/{ticker}", response_model=NewsSocialResponse) +@router.get( + "/{ticker}", + response_model=NewsSocialResponse, + summary="Get news and social media for a ticker", + description=""" + Fetch recent news articles and social media posts for a ticker from multiple sources. + + **News sources**: Yahoo Finance, NewsAPI + **Social sources**: Reddit (r/stocks, r/investing, r/wallstreetbets, r/SecurityAnalysis, r/ValueInvesting) + + Both sources are fetched in parallel. Results are deduplicated and ranked by relevance. + Cached for **10 minutes**. + + **Examples**: + - `GET /news/AAPL` — last 7 days, up to 20 articles + 15 posts + - `GET /news/TSLA?days_back=14&max_articles=50&include_social=false` — news-only, 2 weeks + """, +) @with_cache(namespace="news:full", ttl=600, key_params=["ticker", "days_back", "max_articles", "max_social_posts", "include_social"]) async def get_ticker_news_and_social( ticker: str, @@ -148,7 +165,19 @@ async def get_ticker_news_and_social( ) -@router.get("/{ticker}/news-only", response_model=NewsOnlyResponse) +@router.get( + "/{ticker}/news-only", + response_model=NewsOnlyResponse, + summary="Get news articles for a ticker (no social media)", + description=""" + Faster endpoint that returns only news articles, skipping social media API calls. + + **Sources**: Yahoo Finance, NewsAPI + Cached for **10 minutes**. + + **Example**: `GET /news/NVDA/news-only?days_back=3&max_articles=30` + """, +) @with_cache(namespace="news:news-only", ttl=600, key_params=["ticker", "days_back", "max_articles"]) async def get_ticker_news_only( ticker: str, @@ -217,7 +246,20 @@ async def get_ticker_news_only( ) -@router.get("/{ticker}/social-only", response_model=SocialOnlyResponse) +@router.get( + "/{ticker}/social-only", + response_model=SocialOnlyResponse, + summary="Get social media posts for a ticker", + description=""" + Returns only Reddit posts for a ticker, skipping news API calls. + + **Subreddits**: r/stocks, r/investing, r/wallstreetbets, r/SecurityAnalysis, + r/StockMarket, r/ValueInvesting, r/financialindependence + Cached for **10 minutes**. + + **Example**: `GET /news/GME/social-only?days_back=3&max_social_posts=30` + """, +) @with_cache(namespace="news:social-only", ttl=600, key_params=["ticker", "days_back", "max_social_posts"]) async def get_ticker_social_only( ticker: str, diff --git a/app/api/v1/endpoints/overlay.py b/app/api/v1/endpoints/overlay.py index 433f206..6a37c50 100644 --- a/app/api/v1/endpoints/overlay.py +++ b/app/api/v1/endpoints/overlay.py @@ -14,7 +14,7 @@ from typing import List, Optional from fastapi import APIRouter, Depends, HTTPException, Query from fastapi.responses import Response from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy import select, desc, and_ +from sqlalchemy import select, desc, and_, func from app.core.config import settings from app.core.database import get_db @@ -226,9 +226,26 @@ async def get_top_movers( raise HTTPException(status_code=503, detail="Overlay feature is disabled") cutoff = datetime.now(timezone.utc) - timedelta(hours=24) + + # Get the latest record per symbol, then rank by score + latest_per_symbol = ( + select( + OverlayFeatureRecord.symbol, + func.max(OverlayFeatureRecord.as_of_ts).label("max_ts"), + ) + .where(OverlayFeatureRecord.as_of_ts >= cutoff) + .group_by(OverlayFeatureRecord.symbol) + .subquery() + ) result = await db.execute( select(OverlayFeatureRecord) - .where(OverlayFeatureRecord.as_of_ts >= cutoff) + .join( + latest_per_symbol, + and_( + OverlayFeatureRecord.symbol == latest_per_symbol.c.symbol, + OverlayFeatureRecord.as_of_ts == latest_per_symbol.c.max_ts, + ), + ) .order_by(desc(OverlayFeatureRecord.overlay_score)) .limit(limit) ) @@ -269,9 +286,9 @@ async def admin_health(db: AsyncSession = Depends(get_db)): ) last_job = result.scalars().first() - # Per-source status + # Per-job-type status (collect_all runs all sources; feature_build computes scores) sources = [] - for source_name in ["yahoo_rss", "wikimedia", "youtube", "google_trends", "collect_all", "feature_build"]: + for source_name in ["collect_all", "feature_build"]: result_s = await db.execute( select(OverlayJobLog) .where(OverlayJobLog.job_type == source_name) @@ -312,11 +329,22 @@ async def trigger_pipeline(): asyncio.create_task(_run()) return TriggerPipelineResponse( status="triggered", - message="Overlay pipeline started in background", + message="Overlay pipeline started in background (seeds topic maps, collects data, builds features)", job_ids=[], ) +@router.post( + "/admin/seed-topics", + summary="Seed ThemeTopicMap with default topic mappings", + tags=["overlay-admin"], +) +async def seed_topics(db: AsyncSession = Depends(get_db)): + """Create default ThemeTopicMap entries for all TOP_50_SYMBOLS (safe to re-run; skips existing).""" + inserted = await _pipeline.seed_topic_maps(db) + return {"status": "ok", "inserted": inserted, "message": f"Seeded {inserted} new topic mappings"} + + @router.get( "/admin/job-log", response_model=JobLogResponse, @@ -470,7 +498,7 @@ async def get_youtube( ) for e in sym_events ] - weighted_views = sum(e.view_count * e.channel_weight for e in events_24h) + weighted_views = sum((e.view_count or 0) * (e.channel_weight or 0.5) for e in events_24h) return YouTubeResponse( symbol=symbol, diff --git a/app/api/v1/endpoints/screener.py b/app/api/v1/endpoints/screener.py index a8a5b6b..f70a4df 100644 --- a/app/api/v1/endpoints/screener.py +++ b/app/api/v1/endpoints/screener.py @@ -16,7 +16,7 @@ router = APIRouter() logger = logging.getLogger("app.api.v1.screener") -@router.get("/stocks") +@router.get("/stocks", summary="Screen stocks by financial criteria") @with_cache( namespace="screener:stocks", ttl=300, @@ -128,7 +128,7 @@ async def screen_stocks( ) -@router.get("/fields") +@router.get("/fields", summary="Available screener filter options and valid values") async def get_screener_fields(): """ Return metadata about available screener filter options. diff --git a/app/api/v1/endpoints/stocks.py b/app/api/v1/endpoints/stocks.py index b372a80..9df8eb3 100644 --- a/app/api/v1/endpoints/stocks.py +++ b/app/api/v1/endpoints/stocks.py @@ -85,7 +85,7 @@ async def get_index_constituents( ) -@router.get("/most-active") +@router.get("/most-active", summary="Most actively traded stocks by volume") @with_cache(namespace="stocks:most-active", ttl=3600, key_params=["limit"]) async def get_most_active_stocks( response: Response, @@ -184,7 +184,7 @@ async def get_most_active_stocks( ) -@router.get("/52-week-gainers") +@router.get("/52-week-gainers", summary="Top 52-week gaining stocks") async def get_52week_gainers( limit: Optional[int] = Query(None, ge=1, le=1000, description="Maximum number of stocks to return (1-1000). If not specified, returns first 600 stocks (3 pages) for performance."), max_pages: Optional[int] = Query(3, ge=1, le=10, description="Maximum pages to fetch (1-10). Each page has ~200 stocks. Higher values may cause rate limiting.") @@ -280,7 +280,7 @@ async def get_52week_gainers( ) -@router.get("/trending") +@router.get("/trending", summary="Trending stocks combining most active and 52-week gainers") async def get_trending_stocks( n: Optional[int] = Query(500, ge=1, description="Total number of trending stocks to return after combining most active + gainers (default: 500)"), most_active_limit: Optional[int] = Query(None, ge=1, description="Number of most active stocks to include. If not specified, returns all available stocks (~170)."), diff --git a/app/main.py b/app/main.py index 8b2c38a..c0ced20 100644 --- a/app/main.py +++ b/app/main.py @@ -4,9 +4,9 @@ Main FastAPI application import os from contextlib import asynccontextmanager -from fastapi import FastAPI, HTTPException, Request +from fastapi import FastAPI, HTTPException from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import RedirectResponse, HTMLResponse +from fastapi.responses import RedirectResponse from app.core.config import settings from app.api.v1.api import api_router @@ -62,120 +62,10 @@ app.add_middleware( # Include API router app.include_router(api_router, prefix=settings.API_PREFIX) -# Root documentation endpoint — auto-generated from OpenAPI schema -@app.get("/", response_class=HTMLResponse, include_in_schema=False) -async def root_documentation(request: Request): - try: - schema = request.app.openapi() - paths = schema.get("paths", {}) - api_prefix = settings.API_PREFIX - - # Collect all tags defined in schema (preserves order) - tag_order = [t["name"] for t in schema.get("tags", [])] - - # Group routes by first tag - from collections import defaultdict - tag_routes: dict = defaultdict(list) - untagged: list = [] - for path, methods in paths.items(): - for method, op in methods.items(): - if method.upper() not in ("GET", "POST", "PUT", "DELETE", "PATCH"): - continue - tags = op.get("tags", []) - entry = { - "method": method.upper(), - "path": path, - "summary": op.get("summary", ""), - "deprecated": op.get("deprecated", False), - } - if tags: - tag_routes[tags[0]].append(entry) - else: - untagged.append(entry) - - # Build tag sections — respect schema tag order, then alphabetical remainder - all_tags = tag_order + sorted(t for t in tag_routes if t not in tag_order) - if untagged: - all_tags.append("other") - tag_routes["other"] = untagged - - method_colors = { - "GET": "#61affe", - "POST": "#49cc90", - "PUT": "#fca130", - "DELETE": "#f93e3e", - "PATCH": "#50e3c2", - } - - sections_html = "" - for tag in all_tags: - routes = tag_routes.get(tag, []) - if not routes: - continue - rows = "" - for r in routes: - color = method_colors.get(r["method"], "#999") - deprecated = " style='opacity:0.5;text-decoration:line-through'" if r["deprecated"] else "" - display_path = r["path"].removeprefix(api_prefix) - rows += ( - f"" - f"{r['method']}" - f"{display_path}" - f"{r['summary']}" - f"" - ) - sections_html += f"

{tag}

{rows}
" - - title = schema.get("info", {}).get("title", "API") - version = schema.get("info", {}).get("version", "") - total = sum(len(v) for v in tag_routes.values()) - - html = f""" - - - - - {title} - - - -

🔮 {title}

-
- 📋 Swagger UI - 📖 ReDoc - 💚 Health - ⚙️ OpenAPI JSON -
-

v{version}  ·  Base URL: {api_prefix}  ·  {total} endpoints

- {sections_html} - - -""" - - return HTMLResponse(content=html) - - except Exception: - return RedirectResponse(url=f"{settings.API_PREFIX}/docs") +# Root — redirect to Swagger UI +@app.get("/", include_in_schema=False) +async def root_redirect(): + return RedirectResponse(url=f"{settings.API_PREFIX}/docs") # Additional metadata for OpenAPI app.openapi_tags = [ @@ -234,6 +124,30 @@ app.openapi_tags = [ { "name": "stocks", "description": "Stock market data — most active, 52-week gainers, trending, and index constituents (S&P 500 / Nasdaq 100)" + }, + { + "name": "attention", + "description": "Attention signals — Wikipedia pageview spikes and GDELT news article counts for event-centric backtesting" + }, + { + "name": "attention-admin", + "description": "Attention administrative endpoints — entity resolution, Wikipedia and GDELT data collection" + }, + { + "name": "database", + "description": "Database inspection — record counts, date ranges, raw data browsing, and ETF snapshot history" + }, + { + "name": "fred", + "description": "FRED (Federal Reserve Economic Data) — macroeconomic series via FRED API proxy" + }, + { + "name": "error-logs", + "description": "Error log management — browse and clear server-side error records" + }, + { + "name": "request-logs", + "description": "Request log management — browse API request history and latency records" } ] diff --git a/app/schemas/attention.py b/app/schemas/attention.py index b80a8e2..89e1ff3 100644 --- a/app/schemas/attention.py +++ b/app/schemas/attention.py @@ -4,10 +4,22 @@ Pydantic v2 schemas for the Attention subsystem API. from datetime import date, datetime from typing import Any, Dict, List, Optional -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field class EntityInfo(BaseModel): + model_config = ConfigDict(json_schema_extra={ + "example": { + "ticker": "AAPL", + "canonical_name": "Apple", + "wiki_title": "Apple Inc.", + "gdelt_query": '"Apple" OR "Apple Inc."', + "aliases": ["Apple Inc."], + "resolver_confidence": 0.92, + "is_manual_override": False, + } + }) + ticker: str canonical_name: str = Field(description="Normalized company name with legal suffixes stripped (e.g. 'Apple')") wiki_title: Optional[str] = Field(default=None, description="Matched Wikipedia article title; null if unresolved") @@ -18,6 +30,15 @@ class EntityInfo(BaseModel): class WikiFeatures(BaseModel): + model_config = ConfigDict(json_schema_extra={ + "example": { + "views": 45230, + "baseline_10d": 12400.0, + "spike_10d": 3.65, + "zscore_20d": 4.21, + } + }) + views: Optional[int] = Field(default=None, description="Wikipedia pageviews on the event date") baseline_10d: Optional[float] = Field(default=None, description="Median pageviews over the prior 10 days") spike_10d: Optional[float] = Field(default=None, description="views / baseline_10d; >1 means above-average attention") @@ -25,6 +46,16 @@ class WikiFeatures(BaseModel): class NewsFeatures(BaseModel): + model_config = ConfigDict(json_schema_extra={ + "example": { + "article_count_1d": 18, + "article_count_3d": 52, + "unique_domains_3d": 34, + "us_article_count_3d": 41, + "gdelt_status": "collected", + } + }) + article_count_1d: int = Field(default=0, description="GDELT articles published on the event date") article_count_3d: int = Field(default=0, description="GDELT articles in the event_date ± 1 day window") unique_domains_3d: int = Field(default=0, description="Distinct publisher domains in the 3-day window") @@ -41,6 +72,39 @@ class NewsFeatures(BaseModel): class EventAttentionResponse(BaseModel): + model_config = ConfigDict(json_schema_extra={ + "example": { + "ticker": "AAPL", + "event_date": "2024-02-01", + "entity": { + "ticker": "AAPL", + "canonical_name": "Apple", + "wiki_title": "Apple Inc.", + "gdelt_query": '"Apple" OR "Apple Inc."', + "aliases": ["Apple Inc."], + "resolver_confidence": 0.92, + "is_manual_override": False, + }, + "wiki": { + "views": 45230, + "baseline_10d": 12400.0, + "spike_10d": 3.65, + "zscore_20d": 4.21, + }, + "news": { + "article_count_1d": 18, + "article_count_3d": 52, + "unique_domains_3d": 34, + "us_article_count_3d": 41, + "gdelt_status": "collected", + }, + "metadata": { + "wiki_title": "Apple Inc.", + "resolver_confidence": 0.92, + }, + } + }) + ticker: str event_date: date entity: EntityInfo @@ -50,6 +114,23 @@ class EventAttentionResponse(BaseModel): class EntityResolveResponse(BaseModel): + model_config = ConfigDict(json_schema_extra={ + "example": { + "ticker": "AAPL", + "entity": { + "ticker": "AAPL", + "canonical_name": "Apple", + "wiki_title": "Apple Inc.", + "gdelt_query": '"Apple" OR "Apple Inc."', + "aliases": ["Apple Inc."], + "resolver_confidence": 0.92, + "is_manual_override": False, + }, + "status": "resolved", + "message": "Entity resolved: wiki_title='Apple Inc.' confidence=0.92", + } + }) + ticker: str entity: EntityInfo status: str # "resolved", "already_exists", "failed", "manual_override_skipped" @@ -57,6 +138,16 @@ class EntityResolveResponse(BaseModel): class CollectionStatusResponse(BaseModel): + model_config = ConfigDict(json_schema_extra={ + "example": { + "ticker": "AAPL", + "source": "wiki", + "records_collected": 22, + "date_range": {"event_date": "2024-02-01"}, + "status": "success", + } + }) + ticker: str source: str # "wiki" or "gdelt" records_collected: int diff --git a/app/schemas/filing.py b/app/schemas/filing.py index b4c536b..b5074ae 100644 --- a/app/schemas/filing.py +++ b/app/schemas/filing.py @@ -4,10 +4,20 @@ Pydantic schemas for SEC filing endpoints. from datetime import datetime from typing import Any, Dict, List, Optional -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field class FilingDocumentInfo(BaseModel): + model_config = ConfigDict(json_schema_extra={ + "example": { + "type": "EX-99.1", + "description": "Press Release", + "filename": "ex991pressrelease.htm", + "url": "https://www.sec.gov/Archives/edgar/data/320193/000032019324000006/ex991pressrelease.htm", + "size": "42 KB", + } + }) + type: Optional[str] = None description: Optional[str] = None filename: Optional[str] = None @@ -16,6 +26,18 @@ class FilingDocumentInfo(BaseModel): class FilingSummary(BaseModel): + model_config = ConfigDict(json_schema_extra={ + "example": { + "accession_number": "0000320193-24-000006", + "form_type": "8-K", + "filing_date": "2024-02-01", + "accepted_at": "2024-02-01T21:00:05+00:00", + "primary_document": "a8-k20240201.htm", + "filing_description": "Results of Operations and Financial Condition", + "documents_count": 4, + } + }) + accession_number: str form_type: str filing_date: str @@ -26,6 +48,25 @@ class FilingSummary(BaseModel): class FilingSearchResponse(BaseModel): + model_config = ConfigDict(json_schema_extra={ + "example": { + "ticker": "AAPL", + "filings": [ + { + "accession_number": "0000320193-24-000006", + "form_type": "8-K", + "filing_date": "2024-02-01", + "accepted_at": "2024-02-01T21:00:05+00:00", + "primary_document": "a8-k20240201.htm", + "filing_description": "Results of Operations and Financial Condition", + "documents_count": 4, + } + ], + "total_count": 42, + "metadata": {"limit": 20, "offset": 0, "form_types": ["8-K"]}, + } + }) + ticker: str filings: List[FilingSummary] total_count: int @@ -33,12 +74,46 @@ class FilingSearchResponse(BaseModel): class FilingDocumentListResponse(BaseModel): + model_config = ConfigDict(json_schema_extra={ + "example": { + "accession_number": "0000320193-24-000006", + "documents": [ + { + "type": "8-K", + "description": "8-K", + "filename": "a8-k20240201.htm", + "url": "https://www.sec.gov/Archives/edgar/data/320193/000032019324000006/a8-k20240201.htm", + "size": "8 KB", + }, + { + "type": "EX-99.1", + "description": "Press Release", + "filename": "ex991pressrelease.htm", + "url": "https://www.sec.gov/Archives/edgar/data/320193/000032019324000006/ex991pressrelease.htm", + "size": "42 KB", + }, + ], + "metadata": {"total_documents": 4}, + } + }) + accession_number: str documents: List[FilingDocumentInfo] metadata: dict = Field(default_factory=dict) class ExhibitContentResponse(BaseModel): + model_config = ConfigDict(json_schema_extra={ + "example": { + "accession_number": "0000320193-24-000006", + "exhibit_type": "EX-99.1", + "content": "Apple Reports First Quarter Results...\nCUPERTINO, California — February 1, 2024 — Apple Inc. today announced financial results for its fiscal 2024 first quarter...", + "content_type": "text/html", + "filename": "ex991pressrelease.htm", + "url": "https://www.sec.gov/Archives/edgar/data/320193/000032019324000006/ex991pressrelease.htm", + } + }) + accession_number: str exhibit_type: str content: str @@ -50,6 +125,16 @@ class ExhibitContentResponse(BaseModel): # ── Bulk filing search ────────────────────────────────────────────────────── class BulkFilingSearchRequest(BaseModel): + model_config = ConfigDict(json_schema_extra={ + "example": { + "tickers": ["AAPL", "MSFT", "NVDA"], + "form_type": "8-K", + "start_date": "2024-01-01", + "end_date": "2024-12-31", + "limit_per_ticker": 5, + } + }) + tickers: List[str] = Field(..., min_length=1, max_length=200) form_type: Optional[str] = None start_date: Optional[str] = None @@ -77,6 +162,15 @@ class BulkFilingSearchResponse(BaseModel): # ── Bulk exhibit ──────────────────────────────────────────────────────────── class BulkExhibitRequest(BaseModel): + model_config = ConfigDict(json_schema_extra={ + "example": { + "items": [ + {"accession_number": "0000320193-24-000006", "exhibit_type": "EX-99.1"}, + {"accession_number": "0001045810-24-000010", "exhibit_type": "EX-99.1"}, + ] + } + }) + items: List[Dict[str, str]] = Field(..., min_length=1, max_length=50) # Each item: {"accession_number": "...", "exhibit_type": "EX-99.1"} diff --git a/app/services/overlay/feature_builder.py b/app/services/overlay/feature_builder.py index dfef90b..e16760a 100644 --- a/app/services/overlay/feature_builder.py +++ b/app/services/overlay/feature_builder.py @@ -14,7 +14,9 @@ from app.models.overlay_raw_event import ( OverlayHeadlineEvent, OverlayVideoEvent, OverlayWikiPageview, + OverlayTrendObservation, ) +from app.models.overlay_registry import ThemeTopicMap from app.services.overlay.finra_overlay_loader import FinraOverlayLoader from app.core.overlay_config import ZSCORE_WINDOW_DAYS, WINSOR_LOWER, WINSOR_UPPER @@ -139,13 +141,13 @@ class FeatureBuilder: sym_rows_24h = [r for r in sym_rows if r.published_at >= cutoff_24h] mentions_24h = len(sym_rows_24h) - weighted_views_24h = sum(r.view_count * r.channel_weight for r in sym_rows_24h) + weighted_views_24h = sum((r.view_count or 0) * (r.channel_weight or 0.5) for r in sym_rows_24h) # Daily weighted views for z-score daily_weighted: Dict[str, float] = {} for row in sym_rows: key = _day_key(row.published_at) - daily_weighted[key] = daily_weighted.get(key, 0.0) + row.view_count * row.channel_weight + daily_weighted[key] = daily_weighted.get(key, 0.0) + (row.view_count or 0) * (row.channel_weight or 0.5) hist_values = list(daily_weighted.values()) youtube_influence_z = compute_zscore(weighted_views_24h, hist_values) if hist_values else None @@ -202,6 +204,53 @@ class FeatureBuilder: "wiki_attention_z": wiki_attention_z, } + # ------------------------------------------------------------------ + # Google Trends features + # ------------------------------------------------------------------ + + async def build_trends_features( + self, db: AsyncSession, symbol: str, as_of: datetime + ) -> Dict: + window_cutoff = as_of - timedelta(days=ZSCORE_WINDOW_DAYS) + cutoff_1d = as_of - timedelta(days=1) + + # Find topic IDs mapped to this symbol + topics_result = await db.execute( + select(ThemeTopicMap).where(ThemeTopicMap.active == True) + ) + topics = topics_result.scalars().all() + topic_ids = [t.topic_id for t in topics if symbol in (t.mapped_symbols or [])] + + if not topic_ids: + return {"theme_heat_z": None} + + result = await db.execute( + select( + OverlayTrendObservation.interest_value, + OverlayTrendObservation.observed_at, + ).where( + and_( + OverlayTrendObservation.topic_id.in_(topic_ids), + OverlayTrendObservation.observed_at >= window_cutoff, + ) + ).order_by(OverlayTrendObservation.observed_at) + ) + rows = result.fetchall() + + if not rows: + return {"theme_heat_z": None} + + recent = [r for r in rows if r.observed_at >= cutoff_1d] + current_value = float(sum(r.interest_value for r in recent) / len(recent)) if recent else None + + if current_value is None: + return {"theme_heat_z": None} + + hist_values = [float(r.interest_value) for r in rows] + theme_heat_z = compute_zscore(current_value, hist_values) if len(hist_values) >= 2 else None + + return {"theme_heat_z": theme_heat_z} + # ------------------------------------------------------------------ # FINRA crowding features # ------------------------------------------------------------------ @@ -223,14 +272,14 @@ class FeatureBuilder: headline = await self.build_headline_features(db, symbol, as_of) youtube = await self.build_youtube_features(db, symbol, as_of) wiki = await self.build_wiki_features(db, symbol, as_of) + trends = await self.build_trends_features(db, symbol, as_of) crowding = await self.build_crowding_features(db, symbol) return { **headline, **youtube, **wiki, + **trends, **crowding, "as_of_ts": as_of, - # theme_heat_z comes from Google Trends; left None here (no Trends data yet) - "theme_heat_z": None, } diff --git a/app/services/overlay/overlay_pipeline.py b/app/services/overlay/overlay_pipeline.py index bb5bcaa..e6b78cc 100644 --- a/app/services/overlay/overlay_pipeline.py +++ b/app/services/overlay/overlay_pipeline.py @@ -13,6 +13,7 @@ from sqlalchemy import select, desc from app.core.overlay_config import ONDEMAND_TIMEOUT_SECONDS, FEATURE_STALE_HOURS, TOP_50_SYMBOLS from app.models.overlay_feature import OverlayFeatureRecord, OverlayJobLog +from app.models.overlay_registry import ThemeTopicMap from app.services.overlay.yahoo_rss_adapter import YahooRSSAdapter from app.services.overlay.wikimedia_adapter import WikimediaAdapter from app.services.overlay.youtube_adapter import YouTubeAdapter @@ -47,6 +48,30 @@ class OverlayPipeline: records: int = 0, error: Optional[str] = None, ) -> None: + if status != "running": + # Try to update existing "running" entry rather than creating a duplicate + existing_result = await db.execute( + select(OverlayJobLog) + .where( + OverlayJobLog.job_type == job_type, + OverlayJobLog.started_at == started_at, + OverlayJobLog.status == "running", + ) + .limit(1) + ) + existing = existing_result.scalars().first() + if existing: + existing.status = status + existing.completed_at = datetime.now(timezone.utc) + existing.records_processed = records + existing.error_message = error + try: + await db.commit() + except Exception as e: + logger.warning(f"Failed to update job log: {e}") + await db.rollback() + return + log = OverlayJobLog( job_type=job_type, status=status, @@ -67,26 +92,36 @@ class OverlayPipeline: # ------------------------------------------------------------------ async def collect_all(self, db: AsyncSession) -> Dict: - """Run all data collectors concurrently (source failures are isolated).""" + """Run all data collectors sequentially (source failures are isolated). + + NOTE: Sequential (not concurrent) because SQLAlchemy async sessions do not + allow concurrent operations on the same connection. + """ started = datetime.now(timezone.utc) await self._log_job(db, "collect_all", "running", started) - results = await asyncio.gather( - self.rss.collect(db, symbols=TOP_50_SYMBOLS), - self.wiki.collect(db), - self.youtube.collect(db), - self.trends.collect(db), - return_exceptions=True, - ) - - counts = { - "yahoo_rss": results[0] if isinstance(results[0], int) else 0, - "wikimedia": results[1] if isinstance(results[1], int) else 0, - "youtube": results[2] if isinstance(results[2], int) else 0, - "google_trends": results[3] if isinstance(results[3], int) else 0, + counts: Dict[str, int] = { + "yahoo_rss": 0, + "wikimedia": 0, + "youtube": 0, + "google_trends": 0, } + errors: List[str] = [] + + adapters = [ + ("yahoo_rss", self.rss.collect(db, symbols=TOP_50_SYMBOLS)), + ("wikimedia", self.wiki.collect(db)), + ("youtube", self.youtube.collect(db)), + ("google_trends", self.trends.collect(db)), + ] + for name, coro in adapters: + try: + result = await coro + counts[name] = result if isinstance(result, int) else 0 + except Exception as e: + logger.error(f"Collect error for {name}: {e}") + errors.append(f"{name}: {e}") - errors = [str(r) for r in results if isinstance(r, Exception)] error_str = "; ".join(errors) if errors else None final_status = "partial" if error_str else "completed" @@ -212,12 +247,45 @@ class OverlayPipeline: # Return potentially-stale record rather than None return record + # ------------------------------------------------------------------ + # Topic map seeding + # ------------------------------------------------------------------ + + async def seed_topic_maps(self, db: AsyncSession) -> int: + """Seed ThemeTopicMap with default entries for TOP_50_SYMBOLS if empty. + + Each symbol gets a topic with label "{TICKER} stock" used as Google Trends keyword. + Safe to call repeatedly — skips symbols that already have a mapping. + Returns number of new entries created. + """ + inserted = 0 + for symbol in TOP_50_SYMBOLS: + existing = await db.execute( + select(ThemeTopicMap).where(ThemeTopicMap.topic_id == symbol) + ) + if existing.scalars().first(): + continue + entry = ThemeTopicMap( + topic_id=symbol, + topic_label=f"{symbol} stock", + mapped_symbols=[symbol], + active=True, + ) + db.add(entry) + inserted += 1 + + if inserted: + await db.commit() + logger.info(f"Seeded {inserted} ThemeTopicMap entries") + return inserted + # ------------------------------------------------------------------ # Full pipeline # ------------------------------------------------------------------ async def run_full_pipeline(self, db: AsyncSession) -> Dict: - """Run collect → build feature for all TOP_50 symbols.""" + """Run seed → collect → build feature for all TOP_50 symbols.""" + await self.seed_topic_maps(db) collect_counts = await self.collect_all(db) built = await self.build_features_batch(db) return { diff --git a/app/services/overlay/overlay_scorer.py b/app/services/overlay/overlay_scorer.py index c654cd2..dd45b5d 100644 --- a/app/services/overlay/overlay_scorer.py +++ b/app/services/overlay/overlay_scorer.py @@ -50,8 +50,15 @@ class OverlayScorer: "finra": _zscore_to_01(features.get("crowding_stress_z")), } - # Source presence mask - source_presence_mask = {src: (v is not None) for src, v in normalized.items()} + # Source presence: based on actual raw data, not z-score availability. + # Z-scores require 2+ days of history; a source is "present" if it has any data at all. + source_presence_mask = { + "yahoo": (features.get("headline_count_24h") or 0) > 0, + "youtube": (features.get("youtube_mentions_24h") or 0) > 0, + "wikimedia": features.get("wiki_views_1d") is not None, + "google_trends": normalized.get("google_trends") is not None, + "finra": features.get("short_volume_ratio") is not None, + } # Weighted average across present sources total_weight = 0.0 diff --git a/docker-compose.yml b/docker-compose.yml index b5137d6..8046c2f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,6 +50,7 @@ services: - SEC_EMAIL=example@example.com - ALPACA_API_KEY=${ALPACA_API_KEY:-} - ALPACA_SECRET_KEY=${ALPACA_SECRET_KEY:-} + - GOOGLE_TRENDS_ENABLED=true ports: - "18001:18000" # External:Internal port mapping depends_on: