Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

scale tools dataset to 466k

HenryNdubuaku40ece0793cfe
diff --git a/scripts/build_dataset.py b/scripts/build_dataset.py
index 1e8ded3..5b8b0b7 100644
--- a/scripts/build_dataset.py
+++ b/scripts/build_dataset.py
@@ -13,19 +13,46 @@ import logging
import os
import sys
+# Redirect HF cache to /dev/shm when RAM is plentiful (before importing datasets)
+_shm = "/dev/shm"
+if os.path.isdir(_shm):
+ try:
+ _st = os.statvfs(_shm)
+ if _st.f_bavail * _st.f_frsize >= 200 * 1024**3:
+ _hf_cache = os.path.join(_shm, "needle_hf_cache")
+ os.makedirs(_hf_cache, exist_ok=True)
+ os.environ.setdefault("HF_HOME", _hf_cache)
+ os.environ.setdefault("HF_DATASETS_CACHE", os.path.join(_hf_cache, "datasets"))
+ except OSError:
+ pass
+
import numpy as np
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
from tools_data import load_and_combine
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
logger = logging.getLogger(__name__)
+def _default_output():
+ """Use /dev/shm for output when RAM is plentiful."""
+ shm = "/dev/shm"
+ if os.path.isdir(shm):
+ try:
+ st = os.statvfs(shm)
+ if st.f_bavail * st.f_frsize >= 200 * 1024**3:
+ return os.path.join(shm, "needle_data", "tool_calls_unified")
+ except OSError:
+ pass
+ return "data/tool_calls_unified"
+
+
def main():
parser = argparse.ArgumentParser(description="Build text-only tool-call dataset")
parser.add_argument("--max-samples", type=int, default=None, help="Max samples to include")
- parser.add_argument("--output", type=str, default="data/tool_calls_unified", help="Output directory")
+ parser.add_argument("--output", type=str, default=_default_output(), help="Output directory")
args = parser.parse_args()
logger.info("Loading and combining text datasets...")
diff --git a/scripts/tools_data.py b/scripts/tools_data.py
index 0f8d541..da97119 100755
--- a/scripts/tools_data.py
+++ b/scripts/tools_data.py
@@ -17,6 +17,11 @@ Datasets:
Tool definitions embedded in the system prompt; chat is a flat string with
USER:/A: role markers and <functioncall> tags. Examples with empty answers
(answers=[]) are kept so the model learns when NOT to call functions.
+
+3. Agent-Ark/Toucan-1.5M (OSS, Kimi-K2, Qwen3 subsets)
+ Large-scale tool-calling dataset. Tool calls extracted from 'messages' column
+ (assistant messages with function_call objects), filtered by 'target_tools'.
+ Tool schemas converted from OpenAI format to flat format.
"""
import json
@@ -29,6 +34,9 @@ from datasets import load_dataset, Dataset, concatenate_datasets
DATASETS = {
"xlam-function-calling-60k": "Salesforce/xlam-function-calling-60k",
"glaive-function-calling": "glaiveai/glaive-function-calling-v2",
+ "toucan-oss": ("Agent-Ark/Toucan-1.5M", "OSS"),
+ "toucan-kimi": ("Agent-Ark/Toucan-1.5M", "Kimi-K2"),
+ "toucan-qwen": ("Agent-Ark/Toucan-1.5M", "Qwen3"),
}
@@ -108,9 +116,99 @@ def _parse_glaive_chat(chat):
return query, answers
+def _convert_openai_tools(tools_json):
+ """Convert OpenAI-format tools to flat format.
+
+ OpenAI: [{"type": "function", "function": {"name": ..., "parameters": {"type": "object", "properties": {...}}}}]
+ Flat: [{"name": ..., "description": ..., "parameters": {"param_name": {"type": ..., "description": ...}}}]
+ """
+ try:
+ tools = json.loads(tools_json) if isinstance(tools_json, str) else tools_json
+ except (json.JSONDecodeError, TypeError):
+ return []
+ flat = []
+ for t in tools:
+ if isinstance(t, dict) and "function" in t:
+ fn = t["function"]
+ elif isinstance(t, dict) and "name" in t:
+ fn = t
+ else:
+ continue
+ params = fn.get("parameters") or {}
+ # Unwrap JSON Schema object wrapper → flat param dict
+ if isinstance(params, dict) and "properties" in params:
+ params = params["properties"]
+ flat.append({
+ "name": fn["name"],
+ "description": fn.get("description", ""),
+ "parameters": params,
+ })
+ return flat
+
+
+def _extract_toucan_calls(messages_json, tool_names=None):
+ """Extract tool calls from Toucan messages.
+
+ Collects all function_call entries from assistant messages.
+ If tool_names is provided, only keeps calls whose name matches an available tool.
+ """
+ try:
+ messages = json.loads(messages_json) if isinstance(messages_json, str) else messages_json
+ except (json.JSONDecodeError, TypeError):
+ return []
+ calls = []
+ for msg in messages:
+ if not isinstance(msg, dict):
+ continue
+ if msg.get("role") != "assistant":
+ continue
+ fc = msg.get("function_call")
+ if not fc or not isinstance(fc, dict):
+ continue
+ name = fc.get("name", "")
+ if not name:
+ continue
+ if tool_names is not None and name not in tool_names:
+ continue
+ args = fc.get("arguments", {})
+ if isinstance(args, str):
+ try:
+ args = json.loads(args)
+ except (json.JSONDecodeError, TypeError, ValueError):
+ args = {}
+ if not isinstance(args, dict):
+ args = {}
+ calls.append({"name": name, "arguments": args})
+ return calls
+
+
+def convert_toucan(dataset):
+ rows = []
+ for ex in dataset:
+ query = (ex.get("question") or "").strip()
+ if not query:
+ continue
+ tools = _convert_openai_tools(ex.get("available_tools", "[]"))
+ if not tools:
+ continue
+ tool_names = {t["name"] for t in tools}
+ calls = _extract_toucan_calls(ex.get("messages", "[]"), tool_names)
+ if not calls:
+ continue
+ rows.append({
+ "query": query,
+ "answers": json.dumps(calls),
+ "tools": json.dumps(tools),
+ })
+ return Dataset.from_list(rows)
+
+
CONVERTERS = {
"xlam-function-calling-60k": convert_xlam,
"glaive-function-calling": convert_glaive,
+ "toucan-oss": convert_toucan,
+ "toucan-kimi": convert_toucan,
+ "toucan-qwen": convert_toucan,
}
_PLACEHOLDER_VALUES = {
@@ -250,15 +348,22 @@ def _deduplicate(dataset):
def load_and_combine():
parts = []
for name, hf_id in DATASETS.items():
- print(f"\nLoading {name} ({hf_id})...")
+ if isinstance(hf_id, tuple):
+ repo, subset = hf_id
+ label = f"{repo} [{subset}]"
+ else:
+ repo, subset = hf_id, None
+ label = repo
+ print(f"\nLoading {name} ({label})...")
try:
- ds = load_dataset(hf_id)
+ ds = load_dataset(repo, subset) if subset else load_dataset(repo)
except Exception as e:
print(f" ERROR: {e}")
continue
- print(f" Converting to xlam format...")
- converted = CONVERTERS[name](ds["train"])
+ split = "train" if "train" in ds else list(ds.keys())[0]
+ print(f" Converting to xlam format ({len(ds[split]):,} rows)...")
+ converted = CONVERTERS[name](ds[split])
converted = converted.add_column("source", [name] * len(converted))
before = len(converted)
diff --git a/src/cli.py b/src/cli.py
index 64b73db..04ab8bf 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -1,6 +1,8 @@
import argparse
import sys
+from .data import DEFAULT_MAX_ENC_LEN, DEFAULT_MAX_DEC_LEN, DEFAULT_MAX_GEN_LEN
+
HELP = """Check the readme"""
def main():
@@ -22,15 +24,15 @@ def main():
p.add_argument("--num-kv-heads", type=int, default=8)
p.add_argument("--num-layers", type=int, default=4)
p.add_argument("--num-dec-layers", type=int, default=4)
- p.add_argument("--max-enc-len", type=int, default=1024)
- p.add_argument("--max-dec-len", type=int, default=512)
+ p.add_argument("--max-enc-len", type=int, default=DEFAULT_MAX_ENC_LEN)
+ p.add_argument("--max-dec-len", type=int, default=DEFAULT_MAX_DEC_LEN)
p.add_argument("--max-samples", type=int, default=None)
p.add_argument("--warmup-ratio", type=float, default=0.05)
p.add_argument("--wandb", action="store_true")
p.add_argument("--dtype", type=str, default="bfloat16", choices=["float32", "bfloat16"])
p.add_argument("--checkpoint-dir", type=str, default="checkpoints")
p.add_argument("--seed", type=int, default=42)
- p.add_argument("--eval-every", type=int, default=100)
+ p.add_argument("--eval-every", type=int, default=500)
p.add_argument("--max-eval-samples", type=int, default=None)
p.add_argument("--sparsity-ratio", type=float, default=0.0)
p.add_argument("--group-size", type=int, default=32)
@@ -52,10 +54,10 @@ def main():
p = sub.add_parser("tokenize", add_help=False)
p.add_argument("--max-samples", type=int, default=None,
help="Limit samples per split (for dev/test)")
- p.add_argument("--max-enc-len", type=int, default=1024,
- help="Max encoder sequence length (default: 1024)")
- p.add_argument("--max-dec-len", type=int, default=512,
- help="Max decoder sequence length (default: 512)")
+ p.add_argument("--max-enc-len", type=int, default=DEFAULT_MAX_ENC_LEN,
+ help=f"Max encoder sequence length (default: {DEFAULT_MAX_ENC_LEN})")
+ p.add_argument("--max-dec-len", type=int, default=DEFAULT_MAX_DEC_LEN,
+ help=f"Max decoder sequence length (default: {DEFAULT_MAX_DEC_LEN})")
p = sub.add_parser("run", add_help=False)
p.add_argument("--checkpoint", type=str, required=True)
@@ -69,9 +71,9 @@ def main():
p.add_argument("--checkpoint", type=str, required=True)
p.add_argument("--batch-size", type=int, default=32)
p.add_argument("--max-eval-samples", type=int, default=1000)
- p.add_argument("--max-enc-len", type=int, default=1024)
- p.add_argument("--max-dec-len", type=int, default=512)
- p.add_argument("--max-gen-len", type=int, default=512)
+ p.add_argument("--max-enc-len", type=int, default=DEFAULT_MAX_ENC_LEN)
+ p.add_argument("--max-dec-len", type=int, default=DEFAULT_MAX_DEC_LEN)
+ p.add_argument("--max-gen-len", type=int, default=DEFAULT_MAX_GEN_LEN)
p.add_argument("--tool-call-samples", type=int, default=200,
help="Samples for tool-call accuracy eval (default: 200)")
p.add_argument("--throughput-runs", type=int, default=10)
diff --git a/src/data.py b/src/data.py
index ffc5c31..30ccdd5 100644
--- a/src/data.py
+++ b/src/data.py
@@ -21,28 +21,61 @@ _PROJECT_ROOT = os.path.dirname(os.path.dirname(__file__))
_DISK_CACHE_DIR = os.path.join(_PROJECT_ROOT, ".data_cache")
TOKENIZER_DIR = os.path.join(_PROJECT_ROOT, "tokenizer")
TOKENIZER_PREFIX = os.path.join(TOKENIZER_DIR, "needle")
-LOCAL_UNIFIED_DIR = os.path.join(_PROJECT_ROOT, "data", "tool_calls_unified")
+_DISK_UNIFIED_DIR = os.path.join(_PROJECT_ROOT, "data", "tool_calls_unified")
+_SHM_UNIFIED_DIR = os.path.join("/dev/shm", "needle_data", "tool_calls_unified")
+
+
+def _pick_unified_dir():
+ """Use /dev/shm for unified dataset when RAM is plentiful, else disk."""
+ # Prefer shm if it exists there already or if we have enough RAM
+ if os.path.isdir(_SHM_UNIFIED_DIR) and any(
+ f.endswith(".arrow") for f in os.listdir(_SHM_UNIFIED_DIR)
+ ):
+ return _SHM_UNIFIED_DIR
+ if _shm_available():
+ return _SHM_UNIFIED_DIR
+ return _DISK_UNIFIED_DIR
+
+
+LOCAL_UNIFIED_DIR = _pick_unified_dir()
_MIN_SHM_BYTES = 200 * 1024**3
-def _pick_cache_dir():
- """Use /dev/shm (tmpfs/RAM) when available and large enough, else disk."""
+def _shm_available():
+ """Check if /dev/shm has enough space for RAM-backed storage."""
shm = "/dev/shm"
if os.path.isdir(shm):
try:
st = os.statvfs(shm)
- avail = st.f_bavail * st.f_frsize
- if avail >= _MIN_SHM_BYTES:
- d = os.path.join(shm, "needle_cache")
- os.makedirs(d, exist_ok=True)
- return d
+ return st.f_bavail * st.f_frsize >= _MIN_SHM_BYTES
except OSError:
pass
+ return False
+
+
+def _pick_cache_dir():
+ """Use /dev/shm (tmpfs/RAM) when available and large enough, else disk."""
+ if _shm_available():
+ d = os.path.join("/dev/shm", "needle_cache")
+ os.makedirs(d, exist_ok=True)
+ return d
os.makedirs(_DISK_CACHE_DIR, exist_ok=True)
return _DISK_CACHE_DIR
+def _setup_hf_cache():
+ """Point HuggingFace download cache to /dev/shm when RAM is plentiful."""
+ if _shm_available():
+ hf_cache = os.path.join("/dev/shm", "needle_hf_cache")
+ os.makedirs(hf_cache, exist_ok=True)
+ os.environ.setdefault("HF_HOME", hf_cache)
+ os.environ.setdefault("HF_DATASETS_CACHE", os.path.join(hf_cache, "datasets"))
+
+
+_setup_hf_cache()
+
+
CACHE_DIR = _pick_cache_dir()
PAD_ID = 0
@@ -53,12 +86,17 @@ TOOL_CALL_ID = 4
TRANSCRIBE_ID = 5
TOOLS_ID = 6
+DEFAULT_MAX_ENC_LEN = 1024
+DEFAULT_MAX_DEC_LEN = 512
+DEFAULT_MAX_GEN_LEN = 512
+
_unified_dataset_cache = None
def _load_unified_dataset():
- """Load the unified dataset from local disk.
+ """Load the unified dataset from /dev/shm or disk.
+ Checks /dev/shm first (RAM-backed), then falls back to disk.
Caches the result in memory after first load.
Uses soundfile as the audio decoding backend.
"""
@@ -66,22 +104,22 @@ def _load_unified_dataset():
if _unified_dataset_cache is not None:
return _unified_dataset_cache
- if os.path.exists(LOCAL_UNIFIED_DIR) and any(
- f.endswith(".arrow") for f in os.listdir(LOCAL_UNIFIED_DIR)
- ):
- try:
- ds = load_from_disk(LOCAL_UNIFIED_DIR)
- print(f"Loaded unified dataset from {LOCAL_UNIFIED_DIR} ({len(ds)} rows)")
- _unified_dataset_cache = _set_audio_backend(ds)
- return _unified_dataset_cache
- except Exception as e:
- raise FileNotFoundError(
- f"Failed to load dataset from {LOCAL_UNIFIED_DIR}: {e}. "
- f"Run 'python scripts/build_dataset.py' first."
- )
+ # Try each candidate path: shm first, then disk
+ for path in [_SHM_UNIFIED_DIR, _DISK_UNIFIED_DIR]:
+ if os.path.exists(path) and any(
+ f.endswith(".arrow") for f in os.listdir(path)
+ ):
+ try:
+ ds = load_from_disk(path)
+ print(f"Loaded unified dataset from {path} ({len(ds)} rows)")
+ _unified_dataset_cache = _set_audio_backend(ds)
+ return _unified_dataset_cache
+ except Exception as e:
+ print(f"Warning: failed to load from {path}: {e}")
+ continue
raise FileNotFoundError(
- f"Unified dataset not found at {LOCAL_UNIFIED_DIR}. "
+ f"Unified dataset not found at {_SHM_UNIFIED_DIR} or {_DISK_UNIFIED_DIR}. "
f"Run 'python scripts/build_dataset.py' first."
)
@@ -285,7 +323,7 @@ def _load_cache_metadata(split):
return None
-def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=1024, max_dec_len=512):
+def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_dec_len=DEFAULT_MAX_DEC_LEN):
"""Prepare tool-call encoder-decoder pairs with <tool_call> task token.
Encoder input: [query_tokens..., <tools>, tools_tokens...] truncated to max_enc_len.
diff --git a/src/eval.py b/src/eval.py
index 822cbcc..8d35a83 100644
--- a/src/eval.py
+++ b/src/eval.py
@@ -7,7 +7,7 @@ import jax.numpy as jnp
import numpy as np
import optax
-from .data import get_batches, get_tokenizer, load_tool_calls, prepare_tool_call_pairs
+from .data import get_batches, get_tokenizer, load_tool_calls, prepare_tool_call_pairs, DEFAULT_MAX_ENC_LEN, DEFAULT_MAX_DEC_LEN, DEFAULT_MAX_GEN_LEN
from .model import (
EncoderDecoderTransformer,
TransformerConfig,
@@ -168,7 +168,7 @@ def compute_wer(hypotheses, references):
return total_edits / max(total_ref_words, 1)
-def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=512, max_enc_len=512):
+def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=DEFAULT_MAX_GEN_LEN, max_enc_len=DEFAULT_MAX_ENC_LEN):
"""Generate tool-call predictions and compute structured metrics."""
import json
from .run import generate_batch
@@ -378,9 +378,9 @@ def parse_args():
parser.add_argument("--checkpoint", type=str, default="checkpoints/checkpoint_epoch3.pkl")
parser.add_argument("--batch-size", type=int, default=32)
parser.add_argument("--max-eval-samples", type=int, default=1000)
- parser.add_argument("--max-enc-len", type=int, default=256)
- parser.add_argument("--max-dec-len", type=int, default=1024)
- parser.add_argument("--max-gen-len", type=int, default=512)
+ parser.add_argument("--max-enc-len", type=int, default=DEFAULT_MAX_ENC_LEN)
+ parser.add_argument("--max-dec-len", type=int, default=DEFAULT_MAX_DEC_LEN)
+ parser.add_argument("--max-gen-len", type=int, default=DEFAULT_MAX_GEN_LEN)
parser.add_argument("--throughput-runs", type=int, default=10)
parser.add_argument("--tool-call-samples", type=int, default=200,
help="Samples for tool-call accuracy eval (default: 200)")
diff --git a/src/run.py b/src/run.py
index 7bb18ef..f3534c5 100644
--- a/src/run.py
+++ b/src/run.py
@@ -6,7 +6,7 @@ import jax
import jax.numpy as jnp
import numpy as np
-from .data import get_tokenizer, compute_mel_spectrogram
+from .data import get_tokenizer, compute_mel_spectrogram, DEFAULT_MAX_ENC_LEN, DEFAULT_MAX_GEN_LEN
from .model import (
EncoderDecoderTransformer,
TransformerConfig,
@@ -48,7 +48,7 @@ def load_checkpoint(path):
return params, config
-def _build_encoder_input(tokenizer, query, tools, max_enc_len=1024):
+def _build_encoder_input(tokenizer, query, tools, max_enc_len=DEFAULT_MAX_ENC_LEN):
"""Build encoder input: [query..., <tools>, tools...] truncated to max_enc_len."""
tools_sep_id = tokenizer.tools_token_id
q_toks = tokenizer.encode(query)
@@ -61,7 +61,7 @@ def _build_encoder_input(tokenizer, query, tools, max_enc_len=1024):
return q_toks + [tools_sep_id] + t_toks
-def generate(model, params, tokenizer, query, tools="[]", max_gen_len=512, max_enc_len=1024, seed=0, stream=True, task_token_id=None):
+def generate(model, params, tokenizer, query, tools="[]", max_gen_len=DEFAULT_MAX_GEN_LEN, max_enc_len=DEFAULT_MAX_ENC_LEN, seed=0, stream=True, task_token_id=None):
"""Generate tool-call output.
Encoder: [query_tokens..., <tools>, tools_tokens...] truncated to max_enc_len.
@@ -118,7 +118,7 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=512, max_e
return tokenizer.decode(generated_tokens)
-def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=512, max_enc_len=1024):
+def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DEFAULT_MAX_GEN_LEN, max_enc_len=DEFAULT_MAX_ENC_LEN):
"""Batch-generate tool-call outputs for multiple examples at once.
Encoder: [query_tokens..., <tools>, tools_tokens...] per example, truncated to max_enc_len.
@@ -194,7 +194,7 @@ def load_audio(path, target_sr=16000):
return audio, sr
-def generate_from_audio(model, params, tokenizer, audio_array, sr=16000, tools="[]", max_gen_len=512, seed=0, stream=True):
+def generate_from_audio(model, params, tokenizer, audio_array, sr=16000, tools="[]", max_gen_len=DEFAULT_MAX_GEN_LEN, seed=0, stream=True):
"""Generate tool-call output from audio using the speech encoder pathway.
mel -> encode_speech -> decoder [BOS, <tool_call>, tools_tokens...] -> greedy decode.
diff --git a/src/tokenize_data.py b/src/tokenize_data.py
index 1158123..535ac91 100644
--- a/src/tokenize_data.py
+++ b/src/tokenize_data.py
@@ -13,6 +13,8 @@ import shutil
from .data import (
CACHE_DIR,
+ DEFAULT_MAX_DEC_LEN,
+ DEFAULT_MAX_ENC_LEN,
TOKENIZER_DIR,
_cache_key,
_save_cache_metadata,
@@ -41,8 +43,8 @@ def tokenize(args):
tokenizer = get_tokenizer()
print("\n=== Tokenizing text data ===")
- max_enc_len = getattr(args, "max_enc_len", 1024)
- max_dec_len = getattr(args, "max_dec_len", 512)
+ max_enc_len = getattr(args, "max_enc_len", DEFAULT_MAX_ENC_LEN)
+ max_dec_len = getattr(args, "max_dec_len", DEFAULT_MAX_DEC_LEN)
for split in ("train", "val"):
print(f"\n--- {split} split ---")