You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
139 lines
5.3 KiB
Python
139 lines
5.3 KiB
Python
"""Macro Sync: fetch FRED series → macro_observations table."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import datetime as dt
|
|
import uuid
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.dialects.postgresql import insert
|
|
|
|
from libs.common.config import get_settings
|
|
from libs.common.ids import new_job_run_id
|
|
from libs.common.logging import bind_job_run_id, configure_logging, get_logger
|
|
from libs.db.models import JobRun, MacroObservation, MacroSeries, SyncCheckpoint
|
|
from libs.db.session import get_session
|
|
from libs.oracle_client import FredService, make_oracle_client
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
|
|
async def run_macro_sync(run_id: str) -> dict[str, int]:
|
|
settings = get_settings()
|
|
fred_series_config = settings.get_fred_series()
|
|
stats = {"seen": 0, "written": 0, "errors": 0}
|
|
|
|
async with make_oracle_client() as client:
|
|
svc = FredService(client)
|
|
|
|
async with get_session() as session:
|
|
job = JobRun(
|
|
job_run_id=uuid.UUID(run_id),
|
|
job_name="macro_sync",
|
|
source_name="fred",
|
|
run_date=dt.date.today(),
|
|
status="running",
|
|
)
|
|
session.add(job)
|
|
await session.flush()
|
|
|
|
for series_config in fred_series_config:
|
|
series_id = series_config["id"]
|
|
try:
|
|
# Ensure macro_series row exists
|
|
existing = await session.execute(
|
|
select(MacroSeries).where(MacroSeries.series_id == series_id)
|
|
)
|
|
if existing.scalar_one_or_none() is None:
|
|
ms = MacroSeries(
|
|
series_id=series_id,
|
|
title=series_config.get("title"),
|
|
frequency=series_config.get("frequency"),
|
|
source_name="fred",
|
|
)
|
|
session.add(ms)
|
|
await session.flush()
|
|
|
|
# Fetch observations (last 2 years)
|
|
start = (dt.date.today() - dt.timedelta(days=730)).isoformat()
|
|
response = await svc.get_observations(series_id, start=start)
|
|
stats["seen"] += len(response.observations)
|
|
|
|
rows = []
|
|
for obs in response.observations:
|
|
try:
|
|
val = float(obs.value) if obs.value is not None else None
|
|
except (TypeError, ValueError):
|
|
val = None
|
|
rows.append({
|
|
"series_id": series_id,
|
|
"observation_date": dt.date.fromisoformat(obs.date),
|
|
"value": val,
|
|
"created_at_utc": dt.datetime.now(tz=dt.UTC),
|
|
})
|
|
|
|
if rows:
|
|
stmt = (
|
|
insert(MacroObservation)
|
|
.values(rows)
|
|
.on_conflict_do_nothing(
|
|
constraint="uq_macro_obs_series_date"
|
|
)
|
|
)
|
|
await session.execute(stmt)
|
|
stats["written"] += len(rows)
|
|
|
|
# Update checkpoint
|
|
cp_stmt = (
|
|
insert(SyncCheckpoint)
|
|
.values(
|
|
domain=f"fred:{series_id}",
|
|
last_sync_at_utc=dt.datetime.now(tz=dt.UTC),
|
|
last_sync_params={"start": start},
|
|
status="success",
|
|
created_at_utc=dt.datetime.now(tz=dt.UTC),
|
|
updated_at_utc=dt.datetime.now(tz=dt.UTC),
|
|
)
|
|
.on_conflict_do_update(
|
|
constraint="uq_sync_checkpoints_domain",
|
|
set_={
|
|
"last_sync_at_utc": dt.datetime.now(tz=dt.UTC),
|
|
"status": "success",
|
|
"updated_at_utc": dt.datetime.now(tz=dt.UTC),
|
|
},
|
|
)
|
|
)
|
|
await session.execute(cp_stmt)
|
|
|
|
logger.info("fred_series_synced", series_id=series_id, count=len(rows))
|
|
|
|
except Exception as exc:
|
|
logger.error("fred_sync_error", series_id=series_id, error=str(exc))
|
|
stats["errors"] += 1
|
|
|
|
job.status = "succeeded" if stats["errors"] == 0 else "partial"
|
|
job.finished_at_utc = dt.datetime.now(tz=dt.UTC)
|
|
job.records_seen = stats["seen"]
|
|
job.records_written = stats["written"]
|
|
job.error_count = stats["errors"]
|
|
|
|
logger.info("macro_sync_done", **stats)
|
|
return stats
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description="Macro Sync")
|
|
parser.add_argument("--run-id", default=new_job_run_id())
|
|
args = parser.parse_args()
|
|
|
|
settings = get_settings()
|
|
configure_logging(settings.log_level)
|
|
bind_job_run_id(args.run_id)
|
|
|
|
asyncio.run(run_macro_sync(args.run_id))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|