Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

xlam merge script

HenryNdubuakud18fac152575
diff --git a/scripts/merge_xlam.py b/scripts/merge_xlam.py
new file mode 100644
index 0000000..a0fd596
--- /dev/null
+++ b/scripts/merge_xlam.py
@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+"""Merge Salesforce/xlam-function-calling-60k into the existing unified dataset
+and re-upload to HuggingFace (Cactus-Compute/tool-calls).
+
+Usage:
+ python scripts/merge_xlam.py # merge + upload
+ python scripts/merge_xlam.py --dry-run # merge only, no upload
+ python scripts/merge_xlam.py --max-samples 1000 # limit xlam samples (for testing)
+"""
+
+import argparse
+import json
+import os
+import sys
+
+from datasets import concatenate_datasets, load_dataset, load_from_disk
+
+LOCAL_UNIFIED_DIR = os.path.join(os.path.dirname(__file__), "..", "data", "tool_calls_unified")
+HF_DATASET_REPO = "Cactus-Compute/tool-calls"
+
+
+def load_existing():
+ local = os.path.abspath(LOCAL_UNIFIED_DIR)
+ if os.path.exists(local) and any(f.endswith(".arrow") for f in os.listdir(local)):
+ ds = load_from_disk(local)
+ print(f"Loaded existing dataset from disk: {len(ds)} rows")
+ else:
+ print(f"Downloading existing dataset from {HF_DATASET_REPO}...")
+ ds = load_dataset(HF_DATASET_REPO, split="train", token=True)
+ print(f"Downloaded: {len(ds)} rows")
+ return ds
+
+
+def load_xlam(max_samples=None):
+ print("Loading Salesforce/xlam-function-calling-60k...")
+ ds = load_dataset("Salesforce/xlam-function-calling-60k", split="train")
+ print(f"Loaded xlam: {len(ds)} rows")
+
+ if max_samples and max_samples < len(ds):
+ ds = ds.shuffle(seed=42).select(range(max_samples))
+ print(f"Sampled down to {len(ds)} rows")
+
+ if "id" in ds.column_names:
+ ds = ds.remove_columns(["id"])
+
+ ds = ds.add_column("source", ["xlam-function-calling-60k"] * len(ds))
+ ds = ds.add_column("model", ["human-curated"] * len(ds))
+
+ return ds
+
+
+def check_duplicates(existing, new):
+ """Quick check for query overlap."""
+ existing_queries = set(existing["query"][:10000])
+ new_queries = set(new["query"])
+ overlap = existing_queries & new_queries
+ if overlap:
+ print(f"Warning: {len(overlap)} overlapping queries found (sampling first 10k existing)")
+ else:
+ print("No duplicate queries detected (sampled check)")
+
+
+def main():
+ parser = argparse.ArgumentParser(description="Merge xlam into unified dataset")
+ parser.add_argument("--dry-run", action="store_true", help="Skip upload")
+ parser.add_argument("--max-samples", type=int, default=None, help="Limit xlam samples")
+ args = parser.parse_args()
+
+ existing = load_existing()
+ xlam = load_xlam(args.max_samples)
+
+ xlam = xlam.select_columns(existing.column_names)
+
+ check_duplicates(existing, xlam)
+
+ merged = concatenate_datasets([existing, xlam])
+ print(f"\nMerged dataset: {len(merged)} rows ({len(existing)} existing + {len(xlam)} new)")
+
+ from collections import Counter
+ counts = Counter(merged["source"])
+ print("\nBreakdown by source:")
+ for src, cnt in counts.most_common():
+ print(f" {src}: {cnt}")
+
+ if args.dry_run:
+ print("\n--dry-run: skipping save and upload")
+ return
+
+ import shutil
+ local = os.path.abspath(LOCAL_UNIFIED_DIR)
+ tmp_dir = local + "_tmp"
+ if os.path.exists(tmp_dir):
+ shutil.rmtree(tmp_dir)
+ print(f"\nSaving to {local}...")
+ merged.save_to_disk(tmp_dir)
+ if os.path.exists(local):
+ shutil.rmtree(local)
+ os.rename(tmp_dir, local)
+ print("Saved.")
+
+ # Upload to HuggingFace
+ from huggingface_hub import HfApi
+
+ api = HfApi()
+ api.create_repo(HF_DATASET_REPO, repo_type="dataset", private=False, exist_ok=True)
+ print(f"\nUploading to {HF_DATASET_REPO}...")
+ merged.push_to_hub(HF_DATASET_REPO, token=True)
+ print(f"Upload complete: {HF_DATASET_REPO}")
+
+
+if __name__ == "__main__":
+ main()
\ No newline at end of file