Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
scale tools dataset to 466k
HenryNdubuaku40ece0793cfe
diff --git a/scripts/build_dataset.py b/scripts/build_dataset.pyindex 1e8ded3..5b8b0b7 100644--- a/scripts/build_dataset.py+++ b/scripts/build_dataset.py@@ -13,19 +13,46 @@ import loggingimport osimport sys+# Redirect HF cache to /dev/shm when RAM is plentiful (before importing datasets)+_shm = "/dev/shm"+if os.path.isdir(_shm):+ try:+ _st = os.statvfs(_shm)+ if _st.f_bavail * _st.f_frsize >= 200 * 1024**3:+ _hf_cache = os.path.join(_shm, "needle_hf_cache")+ os.makedirs(_hf_cache, exist_ok=True)+ os.environ.setdefault("HF_HOME", _hf_cache)+ os.environ.setdefault("HF_DATASETS_CACHE", os.path.join(_hf_cache, "datasets"))+ except OSError:+ pass+import numpy as npsys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))from tools_data import load_and_combinelogging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")logger = logging.getLogger(__name__)+def _default_output():+ """Use /dev/shm for output when RAM is plentiful."""+ shm = "/dev/shm"+ if os.path.isdir(shm):+ try:+ st = os.statvfs(shm)+ if st.f_bavail * st.f_frsize >= 200 * 1024**3:+ return os.path.join(shm, "needle_data", "tool_calls_unified")+ except OSError:+ pass+ return "data/tool_calls_unified"++def main():parser = argparse.ArgumentParser(description="Build text-only tool-call dataset")parser.add_argument("--max-samples", type=int, default=None, help="Max samples to include")- parser.add_argument("--output", type=str, default="data/tool_calls_unified", help="Output directory")+ parser.add_argument("--output", type=str, default=_default_output(), help="Output directory")args = parser.parse_args()logger.info("Loading and combining text datasets...")diff --git a/scripts/tools_data.py b/scripts/tools_data.pyindex 0f8d541..da97119 100755--- a/scripts/tools_data.py+++ b/scripts/tools_data.py@@ -17,6 +17,11 @@ Datasets:Tool definitions embedded in the system prompt; chat is a flat string withUSER:/A: role markers and <functioncall> tags. Examples with empty answers(answers=[]) are kept so the model learns when NOT to call functions.++3. Agent-Ark/Toucan-1.5M (OSS, Kimi-K2, Qwen3 subsets)+ Large-scale tool-calling dataset. Tool calls extracted from 'messages' column+ (assistant messages with function_call objects), filtered by 'target_tools'.+ Tool schemas converted from OpenAI format to flat format."""import json@@ -29,6 +34,9 @@ from datasets import load_dataset, Dataset, concatenate_datasetsDATASETS = {"xlam-function-calling-60k": "Salesforce/xlam-function-calling-60k","glaive-function-calling": "glaiveai/glaive-function-calling-v2",+ "toucan-oss": ("Agent-Ark/Toucan-1.5M", "OSS"),+ "toucan-kimi": ("Agent-Ark/Toucan-1.5M", "Kimi-K2"),+ "toucan-qwen": ("Agent-Ark/Toucan-1.5M", "Qwen3"),}@@ -108,9 +116,99 @@ def _parse_glaive_chat(chat):return query, answers+def _convert_openai_tools(tools_json):+ """Convert OpenAI-format tools to flat format.++ OpenAI: [{"type": "function", "function": {"name": ..., "parameters": {"type": "object", "properties": {...}}}}]+ Flat: [{"name": ..., "description": ..., "parameters": {"param_name": {"type": ..., "description": ...}}}]+ """+ try:+ tools = json.loads(tools_json) if isinstance(tools_json, str) else tools_json+ except (json.JSONDecodeError, TypeError):+ return []+ flat = []+ for t in tools:+ if isinstance(t, dict) and "function" in t:+ fn = t["function"]+ elif isinstance(t, dict) and "name" in t:+ fn = t+ else:+ continue+ params = fn.get("parameters") or {}+ # Unwrap JSON Schema object wrapper → flat param dict+ if isinstance(params, dict) and "properties" in params:+ params = params["properties"]+ flat.append({+ "name": fn["name"],+ "description": fn.get("description", ""),+ "parameters": params,+ })+ return flat+++def _extract_toucan_calls(messages_json, tool_names=None):+ """Extract tool calls from Toucan messages.++ Collects all function_call entries from assistant messages.+ If tool_names is provided, only keeps calls whose name matches an available tool.+ """+ try:+ messages = json.loads(messages_json) if isinstance(messages_json, str) else messages_json+ except (json.JSONDecodeError, TypeError):+ return []+ calls = []+ for msg in messages:+ if not isinstance(msg, dict):+ continue+ if msg.get("role") != "assistant":+ continue+ fc = msg.get("function_call")+ if not fc or not isinstance(fc, dict):+ continue+ name = fc.get("name", "")+ if not name:+ continue+ if tool_names is not None and name not in tool_names:+ continue+ args = fc.get("arguments", {})+ if isinstance(args, str):+ try:+ args = json.loads(args)+ except (json.JSONDecodeError, TypeError, ValueError):+ args = {}+ if not isinstance(args, dict):+ args = {}+ calls.append({"name": name, "arguments": args})+ return calls+++def convert_toucan(dataset):+ rows = []+ for ex in dataset:+ query = (ex.get("question") or "").strip()+ if not query:+ continue+ tools = _convert_openai_tools(ex.get("available_tools", "[]"))+ if not tools:+ continue+ tool_names = {t["name"] for t in tools}+ calls = _extract_toucan_calls(ex.get("messages", "[]"), tool_names)+ if not calls:+ continue+ rows.append({+ "query": query,+ "answers": json.dumps(calls),+ "tools": json.dumps(tools),+ })+ return Dataset.from_list(rows)++CONVERTERS = {"xlam-function-calling-60k": convert_xlam,"glaive-function-calling": convert_glaive,+ "toucan-oss": convert_toucan,+ "toucan-kimi": convert_toucan,+ "toucan-qwen": convert_toucan,}_PLACEHOLDER_VALUES = {@@ -250,15 +348,22 @@ def _deduplicate(dataset):def load_and_combine():parts = []for name, hf_id in DATASETS.items():- print(f"\nLoading {name} ({hf_id})...")+ if isinstance(hf_id, tuple):+ repo, subset = hf_id+ label = f"{repo} [{subset}]"+ else:+ repo, subset = hf_id, None+ label = repo+ print(f"\nLoading {name} ({label})...")try:- ds = load_dataset(hf_id)+ ds = load_dataset(repo, subset) if subset else load_dataset(repo)except Exception as e:print(f" ERROR: {e}")continue- print(f" Converting to xlam format...")- converted = CONVERTERS[name](ds["train"])+ split = "train" if "train" in ds else list(ds.keys())[0]+ print(f" Converting to xlam format ({len(ds[split]):,} rows)...")+ converted = CONVERTERS[name](ds[split])converted = converted.add_column("source", [name] * len(converted))before = len(converted)diff --git a/src/cli.py b/src/cli.pyindex 64b73db..04ab8bf 100644--- a/src/cli.py+++ b/src/cli.py@@ -1,6 +1,8 @@import argparseimport sys+from .data import DEFAULT_MAX_ENC_LEN, DEFAULT_MAX_DEC_LEN, DEFAULT_MAX_GEN_LEN+HELP = """Check the readme"""def main():@@ -22,15 +24,15 @@ def main():p.add_argument("--num-kv-heads", type=int, default=8)p.add_argument("--num-layers", type=int, default=4)p.add_argument("--num-dec-layers", type=int, default=4)- p.add_argument("--max-enc-len", type=int, default=1024)- p.add_argument("--max-dec-len", type=int, default=512)+ p.add_argument("--max-enc-len", type=int, default=DEFAULT_MAX_ENC_LEN)+ p.add_argument("--max-dec-len", type=int, default=DEFAULT_MAX_DEC_LEN)p.add_argument("--max-samples", type=int, default=None)p.add_argument("--warmup-ratio", type=float, default=0.05)p.add_argument("--wandb", action="store_true")p.add_argument("--dtype", type=str, default="bfloat16", choices=["float32", "bfloat16"])p.add_argument("--checkpoint-dir", type=str, default="checkpoints")p.add_argument("--seed", type=int, default=42)- p.add_argument("--eval-every", type=int, default=100)+ p.add_argument("--eval-every", type=int, default=500)p.add_argument("--max-eval-samples", type=int, default=None)p.add_argument("--sparsity-ratio", type=float, default=0.0)p.add_argument("--group-size", type=int, default=32)@@ -52,10 +54,10 @@ def main():p = sub.add_parser("tokenize", add_help=False)p.add_argument("--max-samples", type=int, default=None,help="Limit samples per split (for dev/test)")- p.add_argument("--max-enc-len", type=int, default=1024,- help="Max encoder sequence length (default: 1024)")- p.add_argument("--max-dec-len", type=int, default=512,- help="Max decoder sequence length (default: 512)")+ p.add_argument("--max-enc-len", type=int, default=DEFAULT_MAX_ENC_LEN,+ help=f"Max encoder sequence length (default: {DEFAULT_MAX_ENC_LEN})")+ p.add_argument("--max-dec-len", type=int, default=DEFAULT_MAX_DEC_LEN,+ help=f"Max decoder sequence length (default: {DEFAULT_MAX_DEC_LEN})")p = sub.add_parser("run", add_help=False)p.add_argument("--checkpoint", type=str, required=True)@@ -69,9 +71,9 @@ def main():p.add_argument("--checkpoint", type=str, required=True)p.add_argument("--batch-size", type=int, default=32)p.add_argument("--max-eval-samples", type=int, default=1000)- p.add_argument("--max-enc-len", type=int, default=1024)- p.add_argument("--max-dec-len", type=int, default=512)- p.add_argument("--max-gen-len", type=int, default=512)+ p.add_argument("--max-enc-len", type=int, default=DEFAULT_MAX_ENC_LEN)+ p.add_argument("--max-dec-len", type=int, default=DEFAULT_MAX_DEC_LEN)+ p.add_argument("--max-gen-len", type=int, default=DEFAULT_MAX_GEN_LEN)p.add_argument("--tool-call-samples", type=int, default=200,help="Samples for tool-call accuracy eval (default: 200)")p.add_argument("--throughput-runs", type=int, default=10)diff --git a/src/data.py b/src/data.pyindex ffc5c31..30ccdd5 100644--- a/src/data.py+++ b/src/data.py@@ -21,28 +21,61 @@ _PROJECT_ROOT = os.path.dirname(os.path.dirname(__file__))_DISK_CACHE_DIR = os.path.join(_PROJECT_ROOT, ".data_cache")TOKENIZER_DIR = os.path.join(_PROJECT_ROOT, "tokenizer")TOKENIZER_PREFIX = os.path.join(TOKENIZER_DIR, "needle")-LOCAL_UNIFIED_DIR = os.path.join(_PROJECT_ROOT, "data", "tool_calls_unified")+_DISK_UNIFIED_DIR = os.path.join(_PROJECT_ROOT, "data", "tool_calls_unified")+_SHM_UNIFIED_DIR = os.path.join("/dev/shm", "needle_data", "tool_calls_unified")+++def _pick_unified_dir():+ """Use /dev/shm for unified dataset when RAM is plentiful, else disk."""+ # Prefer shm if it exists there already or if we have enough RAM+ if os.path.isdir(_SHM_UNIFIED_DIR) and any(+ f.endswith(".arrow") for f in os.listdir(_SHM_UNIFIED_DIR)+ ):+ return _SHM_UNIFIED_DIR+ if _shm_available():+ return _SHM_UNIFIED_DIR+ return _DISK_UNIFIED_DIR+++LOCAL_UNIFIED_DIR = _pick_unified_dir()_MIN_SHM_BYTES = 200 * 1024**3-def _pick_cache_dir():- """Use /dev/shm (tmpfs/RAM) when available and large enough, else disk."""+def _shm_available():+ """Check if /dev/shm has enough space for RAM-backed storage."""shm = "/dev/shm"if os.path.isdir(shm):try:st = os.statvfs(shm)- avail = st.f_bavail * st.f_frsize- if avail >= _MIN_SHM_BYTES:- d = os.path.join(shm, "needle_cache")- os.makedirs(d, exist_ok=True)- return d+ return st.f_bavail * st.f_frsize >= _MIN_SHM_BYTESexcept OSError:pass+ return False+++def _pick_cache_dir():+ """Use /dev/shm (tmpfs/RAM) when available and large enough, else disk."""+ if _shm_available():+ d = os.path.join("/dev/shm", "needle_cache")+ os.makedirs(d, exist_ok=True)+ return dos.makedirs(_DISK_CACHE_DIR, exist_ok=True)return _DISK_CACHE_DIR+def _setup_hf_cache():+ """Point HuggingFace download cache to /dev/shm when RAM is plentiful."""+ if _shm_available():+ hf_cache = os.path.join("/dev/shm", "needle_hf_cache")+ os.makedirs(hf_cache, exist_ok=True)+ os.environ.setdefault("HF_HOME", hf_cache)+ os.environ.setdefault("HF_DATASETS_CACHE", os.path.join(hf_cache, "datasets"))+++_setup_hf_cache()++CACHE_DIR = _pick_cache_dir()PAD_ID = 0@@ -53,12 +86,17 @@ TOOL_CALL_ID = 4TRANSCRIBE_ID = 5TOOLS_ID = 6+DEFAULT_MAX_ENC_LEN = 1024+DEFAULT_MAX_DEC_LEN = 512+DEFAULT_MAX_GEN_LEN = 512+_unified_dataset_cache = Nonedef _load_unified_dataset():- """Load the unified dataset from local disk.+ """Load the unified dataset from /dev/shm or disk.+ Checks /dev/shm first (RAM-backed), then falls back to disk.Caches the result in memory after first load.Uses soundfile as the audio decoding backend."""@@ -66,22 +104,22 @@ def _load_unified_dataset():if _unified_dataset_cache is not None:return _unified_dataset_cache- if os.path.exists(LOCAL_UNIFIED_DIR) and any(- f.endswith(".arrow") for f in os.listdir(LOCAL_UNIFIED_DIR)- ):- try:- ds = load_from_disk(LOCAL_UNIFIED_DIR)- print(f"Loaded unified dataset from {LOCAL_UNIFIED_DIR} ({len(ds)} rows)")- _unified_dataset_cache = _set_audio_backend(ds)- return _unified_dataset_cache- except Exception as e:- raise FileNotFoundError(- f"Failed to load dataset from {LOCAL_UNIFIED_DIR}: {e}. "- f"Run 'python scripts/build_dataset.py' first."- )+ # Try each candidate path: shm first, then disk+ for path in [_SHM_UNIFIED_DIR, _DISK_UNIFIED_DIR]:+ if os.path.exists(path) and any(+ f.endswith(".arrow") for f in os.listdir(path)+ ):+ try:+ ds = load_from_disk(path)+ print(f"Loaded unified dataset from {path} ({len(ds)} rows)")+ _unified_dataset_cache = _set_audio_backend(ds)+ return _unified_dataset_cache+ except Exception as e:+ print(f"Warning: failed to load from {path}: {e}")+ continueraise FileNotFoundError(- f"Unified dataset not found at {LOCAL_UNIFIED_DIR}. "+ f"Unified dataset not found at {_SHM_UNIFIED_DIR} or {_DISK_UNIFIED_DIR}. "f"Run 'python scripts/build_dataset.py' first.")@@ -285,7 +323,7 @@ def _load_cache_metadata(split):return None-def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=1024, max_dec_len=512):+def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_dec_len=DEFAULT_MAX_DEC_LEN):"""Prepare tool-call encoder-decoder pairs with <tool_call> task token.Encoder input: [query_tokens..., <tools>, tools_tokens...] truncated to max_enc_len.diff --git a/src/eval.py b/src/eval.pyindex 822cbcc..8d35a83 100644--- a/src/eval.py+++ b/src/eval.py@@ -7,7 +7,7 @@ import jax.numpy as jnpimport numpy as npimport optax-from .data import get_batches, get_tokenizer, load_tool_calls, prepare_tool_call_pairs+from .data import get_batches, get_tokenizer, load_tool_calls, prepare_tool_call_pairs, DEFAULT_MAX_ENC_LEN, DEFAULT_MAX_DEC_LEN, DEFAULT_MAX_GEN_LENfrom .model import (EncoderDecoderTransformer,TransformerConfig,@@ -168,7 +168,7 @@ def compute_wer(hypotheses, references):return total_edits / max(total_ref_words, 1)-def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=512, max_enc_len=512):+def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=DEFAULT_MAX_GEN_LEN, max_enc_len=DEFAULT_MAX_ENC_LEN):"""Generate tool-call predictions and compute structured metrics."""import jsonfrom .run import generate_batch@@ -378,9 +378,9 @@ def parse_args():parser.add_argument("--checkpoint", type=str, default="checkpoints/checkpoint_epoch3.pkl")parser.add_argument("--batch-size", type=int, default=32)parser.add_argument("--max-eval-samples", type=int, default=1000)- parser.add_argument("--max-enc-len", type=int, default=256)- parser.add_argument("--max-dec-len", type=int, default=1024)- parser.add_argument("--max-gen-len", type=int, default=512)+ parser.add_argument("--max-enc-len", type=int, default=DEFAULT_MAX_ENC_LEN)+ parser.add_argument("--max-dec-len", type=int, default=DEFAULT_MAX_DEC_LEN)+ parser.add_argument("--max-gen-len", type=int, default=DEFAULT_MAX_GEN_LEN)parser.add_argument("--throughput-runs", type=int, default=10)parser.add_argument("--tool-call-samples", type=int, default=200,help="Samples for tool-call accuracy eval (default: 200)")diff --git a/src/run.py b/src/run.pyindex 7bb18ef..f3534c5 100644--- a/src/run.py+++ b/src/run.py@@ -6,7 +6,7 @@ import jaximport jax.numpy as jnpimport numpy as np-from .data import get_tokenizer, compute_mel_spectrogram+from .data import get_tokenizer, compute_mel_spectrogram, DEFAULT_MAX_ENC_LEN, DEFAULT_MAX_GEN_LENfrom .model import (EncoderDecoderTransformer,TransformerConfig,@@ -48,7 +48,7 @@ def load_checkpoint(path):return params, config-def _build_encoder_input(tokenizer, query, tools, max_enc_len=1024):+def _build_encoder_input(tokenizer, query, tools, max_enc_len=DEFAULT_MAX_ENC_LEN):"""Build encoder input: [query..., <tools>, tools...] truncated to max_enc_len."""tools_sep_id = tokenizer.tools_token_idq_toks = tokenizer.encode(query)@@ -61,7 +61,7 @@ def _build_encoder_input(tokenizer, query, tools, max_enc_len=1024):return q_toks + [tools_sep_id] + t_toks-def generate(model, params, tokenizer, query, tools="[]", max_gen_len=512, max_enc_len=1024, seed=0, stream=True, task_token_id=None):+def generate(model, params, tokenizer, query, tools="[]", max_gen_len=DEFAULT_MAX_GEN_LEN, max_enc_len=DEFAULT_MAX_ENC_LEN, seed=0, stream=True, task_token_id=None):"""Generate tool-call output.Encoder: [query_tokens..., <tools>, tools_tokens...] truncated to max_enc_len.@@ -118,7 +118,7 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=512, max_ereturn tokenizer.decode(generated_tokens)-def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=512, max_enc_len=1024):+def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DEFAULT_MAX_GEN_LEN, max_enc_len=DEFAULT_MAX_ENC_LEN):"""Batch-generate tool-call outputs for multiple examples at once.Encoder: [query_tokens..., <tools>, tools_tokens...] per example, truncated to max_enc_len.@@ -194,7 +194,7 @@ def load_audio(path, target_sr=16000):return audio, sr-def generate_from_audio(model, params, tokenizer, audio_array, sr=16000, tools="[]", max_gen_len=512, seed=0, stream=True):+def generate_from_audio(model, params, tokenizer, audio_array, sr=16000, tools="[]", max_gen_len=DEFAULT_MAX_GEN_LEN, seed=0, stream=True):"""Generate tool-call output from audio using the speech encoder pathway.mel -> encode_speech -> decoder [BOS, <tool_call>, tools_tokens...] -> greedy decode.diff --git a/src/tokenize_data.py b/src/tokenize_data.pyindex 1158123..535ac91 100644--- a/src/tokenize_data.py+++ b/src/tokenize_data.py@@ -13,6 +13,8 @@ import shutilfrom .data import (CACHE_DIR,+ DEFAULT_MAX_DEC_LEN,+ DEFAULT_MAX_ENC_LEN,TOKENIZER_DIR,_cache_key,_save_cache_metadata,@@ -41,8 +43,8 @@ def tokenize(args):tokenizer = get_tokenizer()print("\n=== Tokenizing text data ===")- max_enc_len = getattr(args, "max_enc_len", 1024)- max_dec_len = getattr(args, "max_dec_len", 512)+ max_enc_len = getattr(args, "max_enc_len", DEFAULT_MAX_ENC_LEN)+ max_dec_len = getattr(args, "max_dec_len", DEFAULT_MAX_DEC_LEN)for split in ("train", "val"):print(f"\n--- {split} split ---")