Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
streamline dataset
HenryNdubuaku20fa3e7bf2a8
diff --git a/src/data.py b/src/data.pyindex daf01b0..e8730d6 100644--- a/src/data.py+++ b/src/data.py@@ -16,6 +16,43 @@ from datasets import Audio, Dataset, load_from_diskfrom tqdm import tqdmimport sentencepiece as spm+HF_DATASET_REPO = "Cactus-Compute/tool-calls"+++def download_hf_split(split="train", repo_id=HF_DATASET_REPO):+ """Download a dataset split from HuggingFace by reading parquet directly.++ Bypasses load_dataset() which can fail when the repo README schema+ doesn't match the actual parquet columns (e.g. after adding 'language').+ """+ from huggingface_hub import HfApi, hf_hub_download+ import pyarrow.parquet as pq++ api = HfApi()+ files = api.list_repo_files(repo_id, repo_type="dataset", token=True)+ prefix = f"data/{split}-"+ parquet_files = sorted(f for f in files if f.startswith(prefix) and f.endswith(".parquet"))++ if not parquet_files:+ raise FileNotFoundError(+ f"No parquet files found for split '{split}' in {repo_id} "+ f"(looked for {prefix}*.parquet)"+ )++ tables = []+ for pf in parquet_files:+ local = hf_hub_download(repo_id=repo_id, filename=pf,+ repo_type="dataset", token=True)+ tables.append(pq.read_table(local))++ if len(tables) == 1:+ table = tables[0]+ else:+ import pyarrow as pa+ table = pa.concat_tables(tables)++ return Dataset(table)+import re as _redef to_snake_case(name):@@ -216,9 +253,8 @@ def _load_split_dataset(split="train"):print(f"Warning: failed to load from {local_dir}: {e}")try:- from datasets import load_dataset as _hf_load- print(f"Downloading {hf_split} split from HuggingFace (Cactus-Compute/tool-calls)...")- ds = _hf_load("Cactus-Compute/tool-calls", split=hf_split, token=True)+ print(f"Downloading {hf_split} split from HuggingFace ({HF_DATASET_REPO})...")+ ds = download_hf_split(hf_split)os.makedirs(local_dir, exist_ok=True)ds.save_to_disk(local_dir)print(f"Loaded from HuggingFace -> {local_dir} ({len(ds)} rows)")diff --git a/src/generate_data.py b/src/generate_data.pyindex 69c9611..7b6651c 100644--- a/src/generate_data.py+++ b/src/generate_data.py@@ -1738,7 +1738,8 @@ def _load_existing():print(f"Loaded existing dataset: {len(ds)} rows")else:print(f"Downloading existing dataset from {HF_DATASET_REPO}...")- ds = load_dataset(HF_DATASET_REPO, split="train", token=True)+ from .data import download_hf_split+ ds = download_hf_split("train", HF_DATASET_REPO)# Save locally so subsequent chunks don't re-downloados.makedirs(local, exist_ok=True)ds.save_to_disk(local)diff --git a/src/merge_xlam.py b/src/merge_xlam.pyindex b085daa..08f6354 100644--- a/src/merge_xlam.py+++ b/src/merge_xlam.py@@ -25,8 +25,9 @@ def load_existing():ds = load_from_disk(local)print(f"Loaded existing dataset from disk: {len(ds)} rows")else:+ from .data import download_hf_splitprint(f"Downloading existing dataset from {HF_DATASET_REPO}...")- ds = load_dataset(HF_DATASET_REPO, split="train", token=True)+ ds = download_hf_split("train", HF_DATASET_REPO)print(f"Downloaded: {len(ds)} rows")return dsdiff --git a/src/rebalance_tools.py b/src/rebalance_tools.pyindex fc51a8d..58be96d 100644--- a/src/rebalance_tools.py+++ b/src/rebalance_tools.py@@ -18,7 +18,7 @@ import osimport randomfrom collections import Counter-from datasets import Dataset, concatenate_datasets, load_dataset+from datasets import Dataset, concatenate_datasetsfrom tqdm import tqdmHF_REPO = "Cactus-Compute/tool-calls"@@ -64,8 +64,9 @@ def _trim_tools(tools_str, called_names, target_n, rng):def rebalance(dry_run=False):+ from .data import download_hf_splitprint(f"Loading dataset from {HF_REPO}...")- ds = load_dataset(HF_REPO, split="train", token=True)+ ds = download_hf_split("train", HF_REPO)print(f"Loaded {len(ds):,} examples")rng = random.Random(42)diff --git a/src/split_dataset.py b/src/split_dataset.pyindex 40235a5..79e38d7 100644--- a/src/split_dataset.py+++ b/src/split_dataset.py@@ -12,11 +12,8 @@ import osimport shutilimport tempfile-from datasets import load_dataset--HF_DATASET_REPO = "Cactus-Compute/tool-calls"-VAL_PER_SOURCE = 2500+VAL_PER_SOURCE = 2000SEED = 42@@ -25,10 +22,9 @@ def main(args=None):if args is not None and getattr(args, "val_per_source", None) is not None:val_per_source = args.val_per_source- print(f"Downloading full dataset from {HF_DATASET_REPO}...")- ds = load_dataset(HF_DATASET_REPO, split="train", token=True,- download_mode="force_redownload",- verification_mode="no_checks")+ print(f"Downloading full dataset from {HF_DATASET_REPO} (train split only)...")+ from .data import download_hf_split+ ds = download_hf_split("train", HF_DATASET_REPO)n = len(ds)print(f"Total rows: {n:,}")diff --git a/src/tokenize_data.py b/src/tokenize_data.pyindex d50ede7..2ed6f29 100644--- a/src/tokenize_data.py+++ b/src/tokenize_data.py@@ -70,13 +70,12 @@ def _clear_local_caches():def _download_synth_dataset():"""Download synthesized tool-calling dataset (train + validation) from HuggingFace."""- from datasets import load_dataset+ from .data import download_hf_splitprint("Downloading dataset from HuggingFace (Cactus-Compute/tool-calls)...")for split in ("train", "validation"):try:- ds = load_dataset("Cactus-Compute/tool-calls", split=split, token=True,- verification_mode="no_checks")+ ds = download_hf_split(split)except Exception as e:raise FileNotFoundError(f"Dataset split '{split}' not found on HuggingFace (Cactus-Compute/tool-calls): {e}\n"diff --git a/src/translate_xlam.py b/src/translate_xlam.pyindex a410163..71af90d 100644--- a/src/translate_xlam.py+++ b/src/translate_xlam.py@@ -236,8 +236,9 @@ def main(args):ds = load_from_disk(local)print(f"Loaded existing dataset: {len(ds)} rows")else:+ from .data import download_hf_splitprint(f"Downloading from {HF_DATASET_REPO}...")- ds = load_dataset(HF_DATASET_REPO, split="train", token=True)+ ds = download_hf_split("train", HF_DATASET_REPO)os.makedirs(local, exist_ok=True)ds.save_to_disk(local)print(f"Downloaded: {len(ds)} rows")