Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

updates

HenryNdubuakuda1931a585d1
diff --git a/scripts/generate_data.py b/scripts/generate_data.py
index c2755b3..dde1423 100644
--- a/scripts/generate_data.py
+++ b/scripts/generate_data.py
@@ -1454,7 +1454,7 @@ def generate_batch(client_pool, batch_size, rng, model):
# Only for call types that benefit from multiple tools (not few_tools types)
use_synth = (
call_type not in ("no_tools", "multi_few_tools")
- and rng.random() < 0.15
+ and rng.random() < 0.50
)
if use_synth:
diff --git a/scripts/split_dataset.py b/scripts/split_dataset.py
index 8e58762..bceef65 100644
--- a/scripts/split_dataset.py
+++ b/scripts/split_dataset.py
@@ -1,16 +1,23 @@
"""Split the existing Cactus-Compute/tool-calls dataset into train/validation splits.
-Takes the last 10k rows as validation, rest as train, and re-uploads as a DatasetDict.
+Shuffles the full dataset with a fixed seed, then takes 10k rows as validation.
+This ensures validation is representative of all data sources, not just the latest append.
+Uploads via HfApi to avoid the push_to_hub dataset card bug.
Usage:
python scripts/split_dataset.py
"""
-from datasets import DatasetDict, load_dataset
+import os
+import shutil
+import tempfile
+
+from datasets import load_dataset
HF_DATASET_REPO = "Cactus-Compute/tool-calls"
VAL_SIZE = 10000
+SEED = 42
def main():
@@ -19,6 +26,9 @@ def main():
n = len(ds)
print(f"Total rows: {n:,}")
+ # Shuffle with fixed seed so validation is representative of all sources
+ ds = ds.shuffle(seed=SEED)
+
val_size = min(VAL_SIZE, int(n * 0.1))
train_ds = ds.select(range(n - val_size))
val_ds = ds.select(range(n - val_size, n))
@@ -26,10 +36,52 @@ def main():
print(f"Train: {len(train_ds):,} rows")
print(f"Validation: {len(val_ds):,} rows")
- ds_dict = DatasetDict({"train": train_ds, "validation": val_ds})
+ # Show source distribution in validation
+ from collections import Counter
+ val_sources = Counter(val_ds["source"])
+ print(f"\nValidation source distribution:")
+ for src, cnt in val_sources.most_common():
+ print(f" {src}: {cnt} ({cnt/len(val_ds)*100:.1f}%)")
+
+ # Upload via HfApi (avoids push_to_hub dataset card bug)
+ from huggingface_hub import HfApi, CommitOperationDelete
+
+ api = HfApi()
+ api.create_repo(HF_DATASET_REPO, repo_type="dataset", private=False, exist_ok=True)
+
+ parquet_dir = tempfile.mkdtemp(prefix="needle_split_")
+ print(f"\nExporting to parquet...")
+ train_ds.to_parquet(os.path.join(parquet_dir, "train.parquet"))
+ val_ds.to_parquet(os.path.join(parquet_dir, "validation.parquet"))
+
+ # Delete old data shards
+ files = api.list_repo_files(HF_DATASET_REPO, repo_type="dataset", token=True)
+ old_shards = [f for f in files if f.startswith("data/")]
+ if old_shards:
+ print(f"Deleting {len(old_shards)} old shards...")
+ ops = [CommitOperationDelete(path_in_repo=f) for f in old_shards]
+ api.create_commit(
+ repo_id=HF_DATASET_REPO, repo_type="dataset", operations=ops,
+ commit_message="Remove old shards before split upload", token=True,
+ )
+
+ print(f"Uploading train split ({len(train_ds):,} rows)...")
+ api.upload_file(
+ path_or_fileobj=os.path.join(parquet_dir, "train.parquet"),
+ path_in_repo="data/train-00000-of-00001.parquet",
+ repo_id=HF_DATASET_REPO, repo_type="dataset", token=True,
+ commit_message=f"Upload train split ({len(train_ds):,} rows)",
+ )
+
+ print(f"Uploading validation split ({len(val_ds):,} rows)...")
+ api.upload_file(
+ path_or_fileobj=os.path.join(parquet_dir, "validation.parquet"),
+ path_in_repo="data/validation-00000-of-00001.parquet",
+ repo_id=HF_DATASET_REPO, repo_type="dataset", token=True,
+ commit_message=f"Upload validation split ({len(val_ds):,} rows)",
+ )
- print(f"\nUploading to {HF_DATASET_REPO}...")
- ds_dict.push_to_hub(HF_DATASET_REPO, token=True)
+ shutil.rmtree(parquet_dir)
print("Done.")