You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
40 lines
1.6 KiB
Python
40 lines
1.6 KiB
Python
"""Merge multiple snapshots and re-split them temporally."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
|
|
from libs.common.config import get_settings
|
|
from libs.common.logging import configure_logging
|
|
from libs.export.merged_snapshot import export_merged_snapshot
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description="Merge snapshots into a single temporally-split snapshot")
|
|
parser.add_argument("--source-snapshot-dir", action="append", required=True, help="Source snapshot directory path; repeat for multiple inputs")
|
|
parser.add_argument("--output-dir", default="./data/datasets/snapshots", help="Snapshot output root")
|
|
parser.add_argument("--snapshot-id", required=True, help="New merged snapshot id")
|
|
parser.add_argument("--split-policy", default="temporal_70_15_15", help="Temporal split policy")
|
|
parser.add_argument("--json", action="store_true", help="Print manifest JSON")
|
|
args = parser.parse_args()
|
|
|
|
configure_logging(get_settings().log_level)
|
|
manifest = export_merged_snapshot(
|
|
source_snapshot_dirs=args.source_snapshot_dir,
|
|
output_dir=args.output_dir,
|
|
snapshot_id=args.snapshot_id,
|
|
split_policy=args.split_policy,
|
|
)
|
|
if args.json:
|
|
print(json.dumps(manifest, indent=2))
|
|
else:
|
|
print(f"Snapshot exported: {manifest['snapshot_id']}")
|
|
print(f" Output: {manifest['output_dir']}")
|
|
print(f" Total rows: {manifest['total_rows']}")
|
|
for split, count in manifest["row_counts"].items():
|
|
print(f" {split}: {count} rows")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|