yirugi
/
stock-oracle
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

perf: API 성능 개선 Round 2 — 9개 항목 (P6-P14)

Browse Source 
Tier 1:
- P6: 미들웨어 request_info 지연 추출 (성공 경로에서 body/headers 읽기 제거)
- P7: GZip 압축 미들웨어 (minimum_size=1000, ~4x 압축)
- P8: Redis maxmemory 512MB + allkeys-lru 퇴거 정책

Tier 2:
- P9: _store_ticker_data() batch upsert 전환 (ON CONFLICT DO NOTHING)
- P10: DB 커넥션 풀 증가 (pool_size=20, max_overflow=30, 환경변수 설정)
- P11: Attention 엔드포인트 Redis 캐시 추가 (@with_cache ttl=3600)

Tier 3:
- P12: Dockerfile 멀티 워커 (--workers 4, dev는 --reload 오버라이드)
- P13: aiohttp ClientSession 싱글턴 공유 (4개 파일 7곳 TCP/TLS 재사용)
- P14: Pydantic model_validate → model_construct (bulk 경로 validation 스킵)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
main
I Luk Kim 5 months ago
parent 6a98cb2d94
commit 0752b29a63
13 changed files with 271 additions and 176 deletions
Show Stats Download Patch File Download Diff File

2
Dockerfile
Unescape Escape View File

@ -24,4 +24,4 @@ RUN mkdir -p /app/data
EXPOSE 18000 EXPOSE 18000
# Run the application # Run the application
CMD ["python", "-m", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "18000", "--reload"] CMD ["python", "-m", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "18000", "--workers", "4"]

7
app/api/v1/endpoints/attention.py
Unescape Escape View File

@ -16,11 +16,12 @@ Admin endpoints:
import logging import logging
from datetime import date, timedelta from datetime import date, timedelta
from fastapi import APIRouter, Depends, HTTPException, Query from fastapi import APIRouter, Depends, HTTPException, Query, Response
from sqlalchemy import func, select from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from app.core.database import get_db from app.core.database import get_db
from app.utils.cache import with_cache
from app.models.attention import AttentionFeaturesDaily, CompanyEntityMap, GdeltArticleRaw from app.models.attention import AttentionFeaturesDaily, CompanyEntityMap, GdeltArticleRaw
from app.services.attention.gdelt_collector import GDELT_EARLIEST_DATE from app.services.attention.gdelt_collector import GDELT_EARLIEST_DATE
from app.schemas.attention import ( from app.schemas.attention import (
@ -237,8 +238,10 @@ async def admin_collect_gdelt(
**Example**: `GET /attention/entity/AAPL` **Example**: `GET /attention/entity/AAPL`
""", """,
) )
@with_cache(namespace="attention:entity", ttl=3600, key_params=["ticker"])
async def get_entity( async def get_entity(
ticker: str, ticker: str,
response: Response = None,
db: AsyncSession = Depends(get_db), db: AsyncSession = Depends(get_db),
) -> EntityResolveResponse: ) -> EntityResolveResponse:
ticker = ticker.upper() ticker = ticker.upper()
@ -297,9 +300,11 @@ async def get_entity(
500: {"description": "Feature materialization or collection error"}, 500: {"description": "Feature materialization or collection error"},
}, },
) )
@with_cache(namespace="attention:event", ttl=3600, key_params=["ticker", "event_date"])
async def get_event_attention( async def get_event_attention(
ticker: str, ticker: str,
event_date: date = Query(..., description="Event date in YYYY-MM-DD format"), event_date: date = Query(..., description="Event date in YYYY-MM-DD format"),
response: Response = None,
db: AsyncSession = Depends(get_db), db: AsyncSession = Depends(get_db),
) -> EventAttentionResponse: ) -> EventAttentionResponse:
ticker = ticker.upper() ticker = ticker.upper()

2
app/core/config.py
Unescape Escape View File

@ -41,6 +41,8 @@ class Settings(BaseSettings):
"postgresql+asyncpg://stockoracle:stockoracle2024@localhost:15433/stock_oracle" "postgresql+asyncpg://stockoracle:stockoracle2024@localhost:15433/stock_oracle"
) )
DATABASE_ECHO: bool = False DATABASE_ECHO: bool = False
DB_POOL_SIZE: int = int(os.getenv("DB_POOL_SIZE", "20"))
DB_MAX_OVERFLOW: int = int(os.getenv("DB_MAX_OVERFLOW", "30"))
# Redis # Redis
REDIS_URL: str = os.getenv("REDIS_URL", "redis://localhost:16380/0") REDIS_URL: str = os.getenv("REDIS_URL", "redis://localhost:16380/0")

4
app/core/database.py
Unescape Escape View File

@ -11,8 +11,8 @@ engine = create_async_engine(
settings.DATABASE_URL, settings.DATABASE_URL,
echo=settings.DATABASE_ECHO, echo=settings.DATABASE_ECHO,
future=True, future=True,
pool_size=10, pool_size=settings.DB_POOL_SIZE,
max_overflow=20, max_overflow=settings.DB_MAX_OVERFLOW,
pool_timeout=30, pool_timeout=30,
pool_pre_ping=True, pool_pre_ping=True,
pool_recycle=3600, pool_recycle=3600,

33
app/core/http_client.py
Unescape Escape View File

@ -0,0 +1,33 @@
"""
Shared aiohttp ClientSession singleton.
Reuses a single TCP connection pool across all HTTP clients,
avoiding repeated TCP/TLS handshake overhead per request.
"""
import logging
from typing import Optional
import aiohttp
logger = logging.getLogger(__name__)
_session: Optional[aiohttp.ClientSession] = None
async def get_http_session() -> aiohttp.ClientSession:
"""Get or create the shared aiohttp ClientSession."""
global _session
if _session is None or _session.closed:
_session = aiohttp.ClientSession(
timeout=aiohttp.ClientTimeout(total=30),
)
return _session
async def close_http_session() -> None:
"""Close the shared session. Call on app shutdown."""
global _session
if _session and not _session.closed:
await _session.close()
_session = None

9
app/main.py
Unescape Escape View File

@ -6,6 +6,7 @@ import os
from contextlib import asynccontextmanager from contextlib import asynccontextmanager
from fastapi import FastAPI, HTTPException from fastapi import FastAPI, HTTPException
from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.cors import CORSMiddleware
from starlette.middleware.gzip import GZipMiddleware
from fastapi.responses import RedirectResponse from fastapi.responses import RedirectResponse
from app.core.config import settings from app.core.config import settings
@ -36,6 +37,11 @@ async def lifespan(app: FastAPI):
stop_scheduler() stop_scheduler()
except Exception: except Exception:
pass pass
try:
from app.core.http_client import close_http_session
await close_http_session()
except Exception:
pass
await engine.dispose() await engine.dispose()
# Create FastAPI app # Create FastAPI app
@ -61,6 +67,9 @@ app.add_middleware(
allow_headers=["*"], allow_headers=["*"],
) )
# GZip compression for responses > 1KB
app.add_middleware(GZipMiddleware, minimum_size=1000)
# Include API router # Include API router
app.include_router(api_router, prefix=settings.API_PREFIX) app.include_router(api_router, prefix=settings.API_PREFIX)

29
app/middleware/error_logger.py
Unescape Escape View File

@ -105,8 +105,8 @@ class ErrorLoggingMiddleware(BaseHTTPMiddleware):
start_time = time.time() start_time = time.time()
# Store request details for potential error logging # Store request details for potential error logging
request_info = await self._extract_request_info(request) request_info = self._extract_request_info_minimal(request)
logger.info(f"Processing request {request_id}: {request.method} {request.url.path}") logger.info(f"Processing request {request_id}: {request.method} {request.url.path}")
try: try:
@ -123,7 +123,10 @@ class ErrorLoggingMiddleware(BaseHTTPMiddleware):
# Check if response indicates an error (4xx or 5xx) # Check if response indicates an error (4xx or 5xx)
if response.status_code >= 400: if response.status_code >= 400:
logger.info(f"Error response detected: {response.status_code} for request {request_id}") logger.info(f"Error response detected: {response.status_code} for request {request_id}")
# Full extraction for error logging (body + headers)
request_info = await self._extract_request_info(request)
# Try to capture response body for errors # Try to capture response body for errors
# We need to consume the response body and recreate it # We need to consume the response body and recreate it
from starlette.responses import Response from starlette.responses import Response
@ -183,7 +186,10 @@ class ErrorLoggingMiddleware(BaseHTTPMiddleware):
except Exception as e: except Exception as e:
# Log unexpected errors # Log unexpected errors
response_time_ms = (time.time() - start_time) * 1000 response_time_ms = (time.time() - start_time) * 1000
# Full extraction for error logging
request_info = await self._extract_request_info(request)
await self._log_error( await self._log_error(
request_id=request_id, request_id=request_id,
request_info=request_info, request_info=request_info,
@ -206,8 +212,21 @@ class ErrorLoggingMiddleware(BaseHTTPMiddleware):
headers={"X-Request-ID": request_id} headers={"X-Request-ID": request_id}
) )
def _extract_request_info_minimal(self, request: Request) -> dict:
"""Extract minimal request info (sync, no body read) for success-path logging."""
return {
"endpoint": str(request.url.path),
"method": request.method,
"path": str(request.url),
"query_params": dict(request.query_params) if request.query_params else None,
"request_body": None,
"headers": None,
"user_agent": request.headers.get("user-agent"),
"client_ip": request.client.host if request.client else None,
}
async def _extract_request_info(self, request: Request) -> dict: async def _extract_request_info(self, request: Request) -> dict:
"""Extract request information for logging""" """Extract full request information for error logging"""
# Get request body if present # Get request body if present
body = None body = None

20
app/services/finra_short_volume_service.py
Unescape Escape View File

@ -10,6 +10,8 @@ import aiohttp
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, and_, func, desc from sqlalchemy import select, and_, func, desc
from app.core.http_client import get_http_session
from app.models.finra_short_volume import FinraShortVolume from app.models.finra_short_volume import FinraShortVolume
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -34,15 +36,15 @@ class FinraShortVolumeService:
url = f"{FINRA_CDN_BASE}/{filename}" url = f"{FINRA_CDN_BASE}/{filename}"
try: try:
async with aiohttp.ClientSession() as session: session = await get_http_session()
async with session.get(url, timeout=aiohttp.ClientTimeout(total=30)) as resp: async with session.get(url, timeout=aiohttp.ClientTimeout(total=30)) as resp:
if resp.status == 404: if resp.status == 404:
logger.debug(f"FINRA file not found (holiday/weekend): {filename}") logger.debug(f"FINRA file not found (holiday/weekend): {filename}")
return None return None
resp.raise_for_status() resp.raise_for_status()
text = await resp.text() text = await resp.text()
logger.info(f"FINRA: downloaded {filename} ({len(text)} bytes)") logger.info(f"FINRA: downloaded {filename} ({len(text)} bytes)")
return text return text
except aiohttp.ClientError as e: except aiohttp.ClientError as e:
logger.error(f"FINRA download error for {filename}: {e}") logger.error(f"FINRA download error for {filename}: {e}")
return None return None

196
app/services/news_social_service.py
Unescape Escape View File

@ -10,6 +10,8 @@ Aggregates news and social media data from multiple sources for sentiment analys
import asyncio import asyncio
import aiohttp import aiohttp
import logging import logging
from app.core.http_client import get_http_session
from datetime import datetime, timedelta from datetime import datetime, timedelta
from typing import Dict, List, Optional, Any, Union from typing import Dict, List, Optional, Any, Union
from dataclasses import dataclass from dataclasses import dataclass
@ -326,42 +328,42 @@ class NewsSocialService:
"language": "en" "language": "en"
} }
async with aiohttp.ClientSession() as session: session = await get_http_session()
async with session.get(url, params=params) as response: async with session.get(url, params=params) as response:
if response.status == 200: if response.status == 200:
data = await response.json() data = await response.json()
articles = [] articles = []
for item in data.get('articles', []): for item in data.get('articles', []):
try: try:
# Parse NewsAPI format - make timezone naive for consistency # Parse NewsAPI format - make timezone naive for consistency
published_str = item['publishedAt'].replace('Z', '+00:00') published_str = item['publishedAt'].replace('Z', '+00:00')
published_at = datetime.fromisoformat(published_str).replace(tzinfo=None) published_at = datetime.fromisoformat(published_str).replace(tzinfo=None)
article = NewsArticle( article = NewsArticle(
title=item.get('title', ''), title=item.get('title', ''),
summary=item.get('description', ''), summary=item.get('description', ''),
content=item.get('content', ''), content=item.get('content', ''),
url=item.get('url', ''), url=item.get('url', ''),
source="NewsAPI", source="NewsAPI",
published_at=published_at, published_at=published_at,
author=item.get('author', ''), author=item.get('author', ''),
image_url=item.get('urlToImage', '') image_url=item.get('urlToImage', '')
) )
articles.append(article) articles.append(article)
except Exception as e: except Exception as e:
logger.warning(f"Error parsing NewsAPI article: {e}") logger.warning(f"Error parsing NewsAPI article: {e}")
continue continue
logger.info(f"Retrieved {len(articles)} articles from NewsAPI") logger.info(f"Retrieved {len(articles)} articles from NewsAPI")
return articles return articles
else: else:
error_data = await response.json() error_data = await response.json()
logger.error(f"NewsAPI error {response.status}: {error_data}") logger.error(f"NewsAPI error {response.status}: {error_data}")
raise NewsAPIError(f"NewsAPI returned {response.status}: {error_data}") raise NewsAPIError(f"NewsAPI returned {response.status}: {error_data}")
except Exception as e: except Exception as e:
logger.error(f"Error fetching NewsAPI articles for {ticker}: {e}") logger.error(f"Error fetching NewsAPI articles for {ticker}: {e}")
@ -405,53 +407,53 @@ class NewsSocialService:
"User-Agent": "StockOracle/1.0.0" "User-Agent": "StockOracle/1.0.0"
} }
async with aiohttp.ClientSession() as session: session = await get_http_session()
async with session.get(url, params=params, headers=headers) as response: async with session.get(url, params=params, headers=headers) as response:
if response.status == 200: if response.status == 200:
data = await response.json() data = await response.json()
for item in data.get('data', {}).get('children', []): for item in data.get('data', {}).get('children', []):
try: try:
post_data = item.get('data', {}) post_data = item.get('data', {})
# Filter out posts that are too old # Filter out posts that are too old
created_utc = post_data.get('created_utc', 0) created_utc = post_data.get('created_utc', 0)
post_date = datetime.fromtimestamp(created_utc) post_date = datetime.fromtimestamp(created_utc)
if (datetime.now() - post_date).days > days_back: if (datetime.now() - post_date).days > days_back:
continue continue
# Skip removed/deleted posts # Skip removed/deleted posts
if post_data.get('removed_by_category') or post_data.get('selftext') == '[removed]': if post_data.get('removed_by_category') or post_data.get('selftext') == '[removed]':
continue
post = SocialPost(
title=post_data.get('title', ''),
content=post_data.get('selftext', ''),
url=f"https://reddit.com{post_data.get('permalink', '')}",
platform="Reddit",
author=post_data.get('author', ''),
published_at=post_date,
score=post_data.get('score', 0),
comments_count=post_data.get('num_comments', 0),
upvotes=post_data.get('ups', 0),
downvotes=post_data.get('downs', 0),
subreddit=post_data.get('subreddit', '')
)
all_posts.append(post)
except Exception as e:
logger.warning(f"Error parsing Reddit post: {e}")
continue continue
elif response.status == 401: post = SocialPost(
logger.error("Reddit API authentication failed") title=post_data.get('title', ''),
# Try to refresh token content=post_data.get('selftext', ''),
self._reddit_token = None url=f"https://reddit.com{post_data.get('permalink', '')}",
await self._ensure_reddit_token() platform="Reddit",
else: author=post_data.get('author', ''),
logger.warning(f"Reddit API error for r/{subreddit}: {response.status}") published_at=post_date,
score=post_data.get('score', 0),
comments_count=post_data.get('num_comments', 0),
upvotes=post_data.get('ups', 0),
downvotes=post_data.get('downs', 0),
subreddit=post_data.get('subreddit', '')
)
all_posts.append(post)
except Exception as e:
logger.warning(f"Error parsing Reddit post: {e}")
continue
elif response.status == 401:
logger.error("Reddit API authentication failed")
# Try to refresh token
self._reddit_token = None
await self._ensure_reddit_token()
else:
logger.warning(f"Reddit API error for r/{subreddit}: {response.status}")
except Exception as e: except Exception as e:
logger.warning(f"Error fetching from r/{subreddit}: {e}") logger.warning(f"Error fetching from r/{subreddit}: {e}")
@ -497,21 +499,21 @@ class NewsSocialService:
auth = aiohttp.BasicAuth(self.reddit_client_id, self.reddit_client_secret) auth = aiohttp.BasicAuth(self.reddit_client_id, self.reddit_client_secret)
async with aiohttp.ClientSession() as session: session = await get_http_session()
async with session.post(auth_url, data=auth_data, auth=auth, headers=headers) as response: async with session.post(auth_url, data=auth_data, auth=auth, headers=headers) as response:
if response.status == 200: if response.status == 200:
token_data = await response.json() token_data = await response.json()
self._reddit_token = token_data.get('access_token') self._reddit_token = token_data.get('access_token')
expires_in = token_data.get('expires_in', 3600) expires_in = token_data.get('expires_in', 3600)
self._reddit_token_expiry = datetime.now() + timedelta(seconds=expires_in - 60) self._reddit_token_expiry = datetime.now() + timedelta(seconds=expires_in - 60)
logger.info("Successfully obtained Reddit access token") logger.info("Successfully obtained Reddit access token")
else: else:
error_data = await response.text() error_data = await response.text()
logger.error(f"Reddit auth error {response.status}: {error_data}") logger.error(f"Reddit auth error {response.status}: {error_data}")
raise RedditAPIError(f"Failed to authenticate with Reddit: {response.status}") raise RedditAPIError(f"Failed to authenticate with Reddit: {response.status}")
except Exception as e: except Exception as e:
logger.error(f"Error obtaining Reddit token: {e}") logger.error(f"Error obtaining Reddit token: {e}")

26
app/services/overlay/yahoo_rss_adapter.py
Unescape Escape View File

@ -11,6 +11,8 @@ import aiohttp
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select from sqlalchemy import select
from app.core.http_client import get_http_session
from app.models.overlay_raw_event import OverlayHeadlineEvent from app.models.overlay_raw_event import OverlayHeadlineEvent
from app.services.overlay.entity_resolver import EntityResolver from app.services.overlay.entity_resolver import EntityResolver
from app.core.overlay_config import YAHOO_RSS_FEEDS, TOP_50_SYMBOLS from app.core.overlay_config import YAHOO_RSS_FEEDS, TOP_50_SYMBOLS
@ -33,18 +35,18 @@ class YahooRSSAdapter:
return [] return []
try: try:
async with aiohttp.ClientSession() as session: session = await get_http_session()
headers = { headers = {
"User-Agent": "Mozilla/5.0 StockOracle/1.0", "User-Agent": "Mozilla/5.0 StockOracle/1.0",
"Accept": "application/rss+xml, application/xml, text/xml", "Accept": "application/rss+xml, application/xml, text/xml",
} }
async with session.get( async with session.get(
url, headers=headers, timeout=aiohttp.ClientTimeout(total=30) url, headers=headers, timeout=aiohttp.ClientTimeout(total=30)
) as resp: ) as resp:
if resp.status != 200: if resp.status != 200:
logger.warning(f"Yahoo RSS: non-200 from {url}: {resp.status}") logger.warning(f"Yahoo RSS: non-200 from {url}: {resp.status}")
return [] return []
text = await resp.text() text = await resp.text()
feed = feedparser.parse(text) feed = feedparser.parse(text)
entries = [] entries = []

32
app/services/overlay/youtube_adapter.py
Unescape Escape View File

@ -9,6 +9,8 @@ from typing import Dict, List, Optional
import aiohttp import aiohttp
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from app.core.http_client import get_http_session
from sqlalchemy import select from sqlalchemy import select
from app.core.config import settings from app.core.config import settings
@ -53,15 +55,15 @@ class YouTubeAdapter:
"key": self.api_key, "key": self.api_key,
} }
try: try:
async with aiohttp.ClientSession() as session: session = await get_http_session()
async with session.get( async with session.get(
YOUTUBE_SEARCH_URL, params=params, timeout=aiohttp.ClientTimeout(total=30) YOUTUBE_SEARCH_URL, params=params, timeout=aiohttp.ClientTimeout(total=30)
) as resp: ) as resp:
if resp.status == 403: if resp.status == 403:
logger.warning("YouTube API: quota exceeded or invalid key") logger.warning("YouTube API: quota exceeded or invalid key")
return [] return []
resp.raise_for_status() resp.raise_for_status()
data = await resp.json() data = await resp.json()
return data.get("items", []) return data.get("items", [])
except Exception as e: except Exception as e:
logger.error(f"YouTube search error for channel {channel_id}: {e}") logger.error(f"YouTube search error for channel {channel_id}: {e}")
@ -77,12 +79,12 @@ class YouTubeAdapter:
"key": self.api_key, "key": self.api_key,
} }
try: try:
async with aiohttp.ClientSession() as session: session = await get_http_session()
async with session.get( async with session.get(
YOUTUBE_VIDEOS_URL, params=params, timeout=aiohttp.ClientTimeout(total=30) YOUTUBE_VIDEOS_URL, params=params, timeout=aiohttp.ClientTimeout(total=30)
) as resp: ) as resp:
resp.raise_for_status() resp.raise_for_status()
data = await resp.json() data = await resp.json()
stats = {} stats = {}
for item in data.get("items", []): for item in data.get("items", []):
vid_id = item["id"] vid_id = item["id"]

85
app/services/price_data_service.py
Unescape Escape View File

@ -588,7 +588,12 @@ class PriceDataService:
# Convert to response models # Convert to response models
price_points = [ price_points = [
PriceDataPoint.model_validate(pd) for pd in price_data PriceDataPoint.model_construct(
date=pd.date.date() if isinstance(pd.date, datetime) else pd.date,
open=pd.open, high=pd.high, low=pd.low, close=pd.close,
volume=pd.volume, adjusted_close=pd.adjusted_close,
data_source=pd.data_source.value if hasattr(pd.data_source, 'value') else pd.data_source,
) for pd in price_data
] ]
# Calculate actual date range from returned data # Calculate actual date range from returned data
@ -821,39 +826,48 @@ class PriceDataService:
ticker_data, ticker_data,
interval: str interval: str
): ):
"""Store individual ticker data with optimized batch operations""" """Store individual ticker data using batch upsert (INSERT ... ON CONFLICT DO NOTHING)."""
# Batch check existing dates to avoid individual DB queries from sqlalchemy.dialects.postgresql import insert as pg_insert
existing_dates = await self._get_existing_dates_for_ticker(db, ticker) import uuid as _uuid
new_records = [] def _safe(val):
for date, row in ticker_data.iterrows(): if val is None:
# Convert pandas timestamp to datetime return None
price_date = date.to_pydatetime() try:
v = float(val)
return None if pd.isna(v) else v
except (TypeError, ValueError):
return None
now = datetime.now(timezone.utc)
rows = []
for date_idx, row in ticker_data.iterrows():
price_date = date_idx.to_pydatetime()
if price_date.tzinfo is None: if price_date.tzinfo is None:
price_date = price_date.replace(tzinfo=timezone.utc) price_date = price_date.replace(tzinfo=timezone.utc)
# Skip if already exists rows.append({
if price_date.date() in existing_dates: 'id': _uuid.uuid4(),
continue 'ticker': ticker,
'date': price_date,
# Prepare new record 'open': _safe(row.get('Open')),
price_record = PriceData( 'high': _safe(row.get('High')),
ticker=ticker, 'low': _safe(row.get('Low')),
date=price_date, 'close': _safe(row.get('Close')) or 0.0,
open=float(row.get('Open', 0)) if not pd.isna(row.get('Open')) else None, 'volume': _safe(row.get('Volume')),
high=float(row.get('High', 0)) if not pd.isna(row.get('High')) else None, 'adjusted_close': _safe(row.get('Close')),
low=float(row.get('Low', 0)) if not pd.isna(row.get('Low')) else None, 'data_source': DataSource.YAHOO_FINANCE.value,
close=float(row.get('Close', 0)) if not pd.isna(row.get('Close')) else 0, 'created_at': now,
volume=float(row.get('Volume', 0)) if not pd.isna(row.get('Volume')) else None, 'updated_at': now,
adjusted_close=float(row.get('Close', 0)) if not pd.isna(row.get('Close')) else None, })
data_source=DataSource.YAHOO_FINANCE
) if not rows:
new_records.append(price_record) return
# Batch insert new records stmt = pg_insert(PriceData).values(rows)
if new_records: stmt = stmt.on_conflict_do_nothing(constraint='uq_price_data')
db.add_all(new_records) await db.execute(stmt)
logger.info(f"Added {len(new_records)} new price records for {ticker}") logger.info(f"Batch upserted {len(rows)} price records for {ticker}")
async def _get_existing_dates_for_ticker( async def _get_existing_dates_for_ticker(
self, self,
@ -927,7 +941,12 @@ class PriceDataService:
# Convert to response models # Convert to response models
price_points = [ price_points = [
PriceDataPoint.model_validate(pd) for pd in price_data PriceDataPoint.model_construct(
date=pd.date.date() if isinstance(pd.date, datetime) else pd.date,
open=pd.open, high=pd.high, low=pd.low, close=pd.close,
volume=pd.volume, adjusted_close=pd.adjusted_close,
data_source=pd.data_source.value if hasattr(pd.data_source, 'value') else pd.data_source,
) for pd in price_data
] ]
# Calculate actual date range from returned data # Calculate actual date range from returned data

2
docker-compose.yml
Unescape Escape View File

@ -23,7 +23,7 @@ services:
redis: redis:
image: redis:7-alpine image: redis:7-alpine
container_name: stock_oracle_cache container_name: stock_oracle_cache
command: redis-server --appendonly yes command: redis-server --appendonly yes --maxmemory 512mb --maxmemory-policy allkeys-lru
ports: ports:
- "16380:6379" # Unique port to avoid conflicts - "16380:6379" # Unique port to avoid conflicts
volumes: volumes:

Write Preview
Loading…
Cancel
Save