Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

streamline dataset

HenryNdubuaku20fa3e7bf2a8
diff --git a/src/data.py b/src/data.py
index daf01b0..e8730d6 100644
--- a/src/data.py
+++ b/src/data.py
@@ -16,6 +16,43 @@ from datasets import Audio, Dataset, load_from_disk
from tqdm import tqdm
import sentencepiece as spm
+HF_DATASET_REPO = "Cactus-Compute/tool-calls"
+
+
+def download_hf_split(split="train", repo_id=HF_DATASET_REPO):
+ """Download a dataset split from HuggingFace by reading parquet directly.
+
+ Bypasses load_dataset() which can fail when the repo README schema
+ doesn't match the actual parquet columns (e.g. after adding 'language').
+ """
+ from huggingface_hub import HfApi, hf_hub_download
+ import pyarrow.parquet as pq
+
+ api = HfApi()
+ files = api.list_repo_files(repo_id, repo_type="dataset", token=True)
+ prefix = f"data/{split}-"
+ parquet_files = sorted(f for f in files if f.startswith(prefix) and f.endswith(".parquet"))
+
+ if not parquet_files:
+ raise FileNotFoundError(
+ f"No parquet files found for split '{split}' in {repo_id} "
+ f"(looked for {prefix}*.parquet)"
+ )
+
+ tables = []
+ for pf in parquet_files:
+ local = hf_hub_download(repo_id=repo_id, filename=pf,
+ repo_type="dataset", token=True)
+ tables.append(pq.read_table(local))
+
+ if len(tables) == 1:
+ table = tables[0]
+ else:
+ import pyarrow as pa
+ table = pa.concat_tables(tables)
+
+ return Dataset(table)
+
import re as _re
def to_snake_case(name):
@@ -216,9 +253,8 @@ def _load_split_dataset(split="train"):
print(f"Warning: failed to load from {local_dir}: {e}")
try:
- from datasets import load_dataset as _hf_load
- print(f"Downloading {hf_split} split from HuggingFace (Cactus-Compute/tool-calls)...")
- ds = _hf_load("Cactus-Compute/tool-calls", split=hf_split, token=True)
+ print(f"Downloading {hf_split} split from HuggingFace ({HF_DATASET_REPO})...")
+ ds = download_hf_split(hf_split)
os.makedirs(local_dir, exist_ok=True)
ds.save_to_disk(local_dir)
print(f"Loaded from HuggingFace -> {local_dir} ({len(ds)} rows)")
diff --git a/src/generate_data.py b/src/generate_data.py
index 69c9611..7b6651c 100644
--- a/src/generate_data.py
+++ b/src/generate_data.py
@@ -1738,7 +1738,8 @@ def _load_existing():
print(f"Loaded existing dataset: {len(ds)} rows")
else:
print(f"Downloading existing dataset from {HF_DATASET_REPO}...")
- ds = load_dataset(HF_DATASET_REPO, split="train", token=True)
+ from .data import download_hf_split
+ ds = download_hf_split("train", HF_DATASET_REPO)
# Save locally so subsequent chunks don't re-download
os.makedirs(local, exist_ok=True)
ds.save_to_disk(local)
diff --git a/src/merge_xlam.py b/src/merge_xlam.py
index b085daa..08f6354 100644
--- a/src/merge_xlam.py
+++ b/src/merge_xlam.py
@@ -25,8 +25,9 @@ def load_existing():
ds = load_from_disk(local)
print(f"Loaded existing dataset from disk: {len(ds)} rows")
else:
+ from .data import download_hf_split
print(f"Downloading existing dataset from {HF_DATASET_REPO}...")
- ds = load_dataset(HF_DATASET_REPO, split="train", token=True)
+ ds = download_hf_split("train", HF_DATASET_REPO)
print(f"Downloaded: {len(ds)} rows")
return ds
diff --git a/src/rebalance_tools.py b/src/rebalance_tools.py
index fc51a8d..58be96d 100644
--- a/src/rebalance_tools.py
+++ b/src/rebalance_tools.py
@@ -18,7 +18,7 @@ import os
import random
from collections import Counter
-from datasets import Dataset, concatenate_datasets, load_dataset
+from datasets import Dataset, concatenate_datasets
from tqdm import tqdm
HF_REPO = "Cactus-Compute/tool-calls"
@@ -64,8 +64,9 @@ def _trim_tools(tools_str, called_names, target_n, rng):
def rebalance(dry_run=False):
+ from .data import download_hf_split
print(f"Loading dataset from {HF_REPO}...")
- ds = load_dataset(HF_REPO, split="train", token=True)
+ ds = download_hf_split("train", HF_REPO)
print(f"Loaded {len(ds):,} examples")
rng = random.Random(42)
diff --git a/src/split_dataset.py b/src/split_dataset.py
index 40235a5..79e38d7 100644
--- a/src/split_dataset.py
+++ b/src/split_dataset.py
@@ -12,11 +12,8 @@ import os
import shutil
import tempfile
-from datasets import load_dataset
-
-
HF_DATASET_REPO = "Cactus-Compute/tool-calls"
-VAL_PER_SOURCE = 2500
+VAL_PER_SOURCE = 2000
SEED = 42
@@ -25,10 +22,9 @@ def main(args=None):
if args is not None and getattr(args, "val_per_source", None) is not None:
val_per_source = args.val_per_source
- print(f"Downloading full dataset from {HF_DATASET_REPO}...")
- ds = load_dataset(HF_DATASET_REPO, split="train", token=True,
- download_mode="force_redownload",
- verification_mode="no_checks")
+ print(f"Downloading full dataset from {HF_DATASET_REPO} (train split only)...")
+ from .data import download_hf_split
+ ds = download_hf_split("train", HF_DATASET_REPO)
n = len(ds)
print(f"Total rows: {n:,}")
diff --git a/src/tokenize_data.py b/src/tokenize_data.py
index d50ede7..2ed6f29 100644
--- a/src/tokenize_data.py
+++ b/src/tokenize_data.py
@@ -70,13 +70,12 @@ def _clear_local_caches():
def _download_synth_dataset():
"""Download synthesized tool-calling dataset (train + validation) from HuggingFace."""
- from datasets import load_dataset
+ from .data import download_hf_split
print("Downloading dataset from HuggingFace (Cactus-Compute/tool-calls)...")
for split in ("train", "validation"):
try:
- ds = load_dataset("Cactus-Compute/tool-calls", split=split, token=True,
- verification_mode="no_checks")
+ ds = download_hf_split(split)
except Exception as e:
raise FileNotFoundError(
f"Dataset split '{split}' not found on HuggingFace (Cactus-Compute/tool-calls): {e}\n"
diff --git a/src/translate_xlam.py b/src/translate_xlam.py
index a410163..71af90d 100644
--- a/src/translate_xlam.py
+++ b/src/translate_xlam.py
@@ -236,8 +236,9 @@ def main(args):
ds = load_from_disk(local)
print(f"Loaded existing dataset: {len(ds)} rows")
else:
+ from .data import download_hf_split
print(f"Downloading from {HF_DATASET_REPO}...")
- ds = load_dataset(HF_DATASET_REPO, split="train", token=True)
+ ds = download_hf_split("train", HF_DATASET_REPO)
os.makedirs(local, exist_ok=True)
ds.save_to_disk(local)
print(f"Downloaded: {len(ds)} rows")