Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
curriculum training + irrelevant validation
HenryNdubuaku7732196ccaf0
diff --git a/scripts/tools_data.py b/scripts/tools_data.pyindex da97119..bd8cd2e 100755--- a/scripts/tools_data.py+++ b/scripts/tools_data.py@@ -22,9 +22,16 @@ Datasets:Large-scale tool-calling dataset. Tool calls extracted from 'messages' column(assistant messages with function_call objects), filtered by 'target_tools'.Tool schemas converted from OpenAI format to flat format.++4. Team-ACE/ToolACE+ Synthetic tool-calling dataset (11.3k) with dual-layer verification.+ Tool definitions in system prompt JSON; calls use bracket notation+ [func(key=val)]. Grounding filter skipped (pre-verified)."""+import astimport json+import randomimport refrom collections import Counter@@ -37,6 +44,9 @@ DATASETS = {"toucan-oss": ("Agent-Ark/Toucan-1.5M", "OSS"),"toucan-kimi": ("Agent-Ark/Toucan-1.5M", "Kimi-K2"),"toucan-qwen": ("Agent-Ark/Toucan-1.5M", "Qwen3"),+ "toucan-sft": ("Agent-Ark/Toucan-1.5M", "SFT"),+ "toolace": "Team-ACE/ToolACE",+ "nemotron-tc": ("nvidia/Nemotron-Agentic-v1", None, "tool_calling"),}@@ -188,7 +198,7 @@ def convert_toucan(dataset):query = (ex.get("question") or "").strip()if not query:continue- tools = _convert_openai_tools(ex.get("available_tools", "[]"))+ tools = _convert_openai_tools(ex.get("available_tools") or ex.get("tools") or "[]")if not tools:continuetool_names = {t["name"] for t in tools}@@ -203,12 +213,277 @@ def convert_toucan(dataset):return Dataset.from_list(rows)+def _extract_toolace_tools(system_text):+ """Extract JSON tool definitions array from ToolACE system prompt."""+ start = system_text.find('[{')+ if start == -1:+ return []+ depth = 0+ for i in range(start, len(system_text)):+ if system_text[i] == '[':+ depth += 1+ elif system_text[i] == ']':+ depth -= 1+ if depth == 0:+ try:+ return json.loads(system_text[start:i + 1])+ except json.JSONDecodeError:+ return []+ return []+++def _split_bracket_args(s, sep=','):+ """Split on separator respecting nested brackets and quoted strings."""+ parts = []+ depth = 0+ in_str = False+ str_char = None+ current = []+ for i, ch in enumerate(s):+ if in_str:+ current.append(ch)+ if ch == str_char and (i == 0 or s[i - 1] != '\\'):+ in_str = False+ elif ch in ('"', "'"):+ in_str = True+ str_char = ch+ current.append(ch)+ elif ch in ('(', '[', '{'):+ depth += 1+ current.append(ch)+ elif ch in (')', ']', '}'):+ depth -= 1+ current.append(ch)+ elif ch == sep and depth == 0:+ parts.append(''.join(current))+ current = []+ else:+ current.append(ch)+ if current:+ parts.append(''.join(current))+ return parts+++def _parse_bracket_value(s):+ """Parse a value string from bracket notation arguments."""+ s = s.strip()+ if not s:+ return s+ try:+ return json.loads(s)+ except (json.JSONDecodeError, ValueError):+ pass+ if s == 'True':+ return True+ if s == 'False':+ return False+ if s in ('None', 'null'):+ return None+ try:+ return ast.literal_eval(s)+ except (ValueError, SyntaxError):+ pass+ return s+++def _parse_toolace_calls(text, tool_names):+ """Parse ToolACE bracket notation into tool calls using known names."""+ text = text.strip()+ if not text.startswith('[') or not text.endswith(']'):+ return []+ inner = text[1:-1].strip()+ if not inner:+ return []++ sorted_names = sorted(tool_names, key=len, reverse=True)+ calls = []+ pos = 0++ while pos < len(inner):+ while pos < len(inner) and inner[pos] in (' ', ',', '\n', '\t'):+ pos += 1+ if pos >= len(inner):+ break++ matched = None+ for name in sorted_names:+ if inner[pos:pos + len(name)] == name:+ rest = inner[pos + len(name):].lstrip()+ if rest and rest[0] == '(':+ matched = name+ break++ if not matched:+ break++ pos += len(matched)+ while pos < len(inner) and inner[pos] == ' ':+ pos += 1+ if pos >= len(inner) or inner[pos] != '(':+ break++ depth = 0+ in_str = False+ str_char = None+ end = pos+ while end < len(inner):+ ch = inner[end]+ if in_str:+ if ch == str_char and (end == 0 or inner[end - 1] != '\\'):+ in_str = False+ elif ch in ('"', "'"):+ in_str = True+ str_char = ch+ elif ch == '(':+ depth += 1+ elif ch == ')':+ depth -= 1+ if depth == 0:+ break+ end += 1++ args_str = inner[pos + 1:end].strip()+ arguments = {}+ if args_str:+ for pair in _split_bracket_args(args_str, ','):+ pair = pair.strip()+ eq = pair.find('=')+ if eq < 0:+ continue+ key = pair[:eq].strip()+ val = _parse_bracket_value(pair[eq + 1:])+ arguments[key] = val++ calls.append({"name": matched, "arguments": arguments})+ pos = end + 1++ return calls+++def convert_toolace(dataset):+ rows = []+ for ex in dataset:+ system = ex.get("system", "")+ convs = ex.get("conversations", [])++ raw_tools = _extract_toolace_tools(system)+ tools = _convert_openai_tools(raw_tools)+ if not tools:+ continue++ tool_names = {t["name"] for t in tools}++ query = None+ assistant_text = None+ for msg in convs:+ role = msg.get("from", "")+ val = (msg.get("value") or "").strip()+ if role == "user" and query is None:+ query = val+ elif role == "assistant" and assistant_text is None:+ assistant_text = val++ if not query:+ continue++ calls = []+ if assistant_text:+ calls = _parse_toolace_calls(assistant_text, tool_names)++ rows.append({+ "query": query,+ "answers": json.dumps(calls),+ "tools": json.dumps(tools),+ })+ return Dataset.from_list(rows)+++def _extract_nemotron_calls(messages, tool_names=None):+ """Extract tool calls from Nemotron messages (OpenAI tool_calls format)."""+ calls = []+ for msg in messages:+ if not isinstance(msg, dict) or msg.get("role") != "assistant":+ continue++ # OpenAI tool_calls format (newer)+ for tc in (msg.get("tool_calls") or []):+ if not isinstance(tc, dict):+ continue+ fn = tc.get("function", tc)+ name = fn.get("name", "")+ if not name:+ continue+ if tool_names and name not in tool_names:+ continue+ args = fn.get("arguments", {})+ if isinstance(args, str):+ try:+ args = json.loads(args)+ except (json.JSONDecodeError, TypeError):+ args = {}+ if not isinstance(args, dict):+ args = {}+ calls.append({"name": name, "arguments": args})++ # Fallback: function_call format (older OpenAI)+ fc = msg.get("function_call")+ if fc and isinstance(fc, dict):+ name = fc.get("name", "")+ if name and (not tool_names or name in tool_names):+ args = fc.get("arguments", {})+ if isinstance(args, str):+ try:+ args = json.loads(args)+ except (json.JSONDecodeError, TypeError):+ args = {}+ if not isinstance(args, dict):+ args = {}+ calls.append({"name": name, "arguments": args})++ return calls+++def convert_nemotron(dataset):+ rows = []+ for ex in dataset:+ tools_raw = ex.get("tools") or []+ tools = _convert_openai_tools(tools_raw)+ if not tools:+ continue++ tool_names = {t["name"] for t in tools}+ messages = ex.get("messages") or []++ query = None+ for msg in messages:+ if isinstance(msg, dict) and msg.get("role") == "user":+ query = (msg.get("content") or "").strip()+ break++ if not query:+ continue++ calls = _extract_nemotron_calls(messages, tool_names)+ if not calls:+ continue++ rows.append({+ "query": query,+ "answers": json.dumps(calls),+ "tools": json.dumps(tools),+ })+ return Dataset.from_list(rows)++CONVERTERS = {"xlam-function-calling-60k": convert_xlam,"glaive-function-calling": convert_glaive,"toucan-oss": convert_toucan,"toucan-kimi": convert_toucan,"toucan-qwen": convert_toucan,+ "toucan-sft": convert_toucan,+ "toolace": convert_toolace,+ "nemotron-tc": convert_nemotron,}_PLACEHOLDER_VALUES = {@@ -331,6 +606,10 @@ def _answer_uses_valid_params(ex):return True+# Sources with their own verification — skip grounding check+_VERIFIED_SOURCES = {"toolace"}++def _deduplicate(dataset):"""Remove duplicate queries, keeping the first occurrence."""seen = set()@@ -345,32 +624,100 @@ def _deduplicate(dataset):return dataset.select(keep)+def _augment_irrelevance(dataset, ratio=0.1, seed=42):+ """Create irrelevance samples by swapping tools with unrelated queries.++ For a fraction of examples with tool calls, replace the available tools+ with tools from a different example and set the answer to [] (no calls).+ Ensures no overlap between the query's needed tools and the donor tools.+ """+ rng = random.Random(seed)++ all_answers = dataset["answers"]+ all_tools = dataset["tools"]+ all_queries = dataset["query"]++ with_calls = []+ call_names = {}+ tool_names_by_idx = {}++ for i, ans_str in enumerate(all_answers):+ try:+ answers = json.loads(ans_str)+ except (json.JSONDecodeError, TypeError):+ continue+ if not answers:+ continue+ with_calls.append(i)+ call_names[i] = {c["name"] for c in answers if isinstance(c, dict)}+ try:+ tools = json.loads(all_tools[i])+ tool_names_by_idx[i] = {t["name"] for t in tools if isinstance(t, dict)}+ except (json.JSONDecodeError, TypeError):+ tool_names_by_idx[i] = set()++ n_aug = int(len(with_calls) * ratio)+ selected = rng.sample(with_calls, min(n_aug, len(with_calls)))++ rows = []+ for idx in selected:+ orig_call_names = call_names[idx]+ for _ in range(5):+ donor = rng.choice(with_calls)+ if donor == idx:+ continue+ # Ensure none of the query's needed tools appear in donor's tool set+ if not (orig_call_names & tool_names_by_idx.get(donor, set())):+ rows.append({+ "query": all_queries[idx],+ "answers": "[]",+ "tools": all_tools[donor],+ "source": "irrelevance-aug",+ })+ break++ if rows:+ print(f"\nIrrelevance augmentation: {len(rows)} samples "+ f"({len(rows)/len(dataset)*100:.1f}% of dataset)")+ return concatenate_datasets([dataset, Dataset.from_list(rows)])+ return dataset++def load_and_combine():parts = []for name, hf_id in DATASETS.items():+ split_override = Noneif isinstance(hf_id, tuple):- repo, subset = hf_id- label = f"{repo} [{subset}]"+ if len(hf_id) == 3:+ repo, subset, split_override = hf_id+ else:+ repo, subset = hf_id+ label = f"{repo}" + (f" [{subset}]" if subset else "")else:repo, subset = hf_id, Nonelabel = repoprint(f"\nLoading {name} ({label})...")try:- ds = load_dataset(repo, subset) if subset else load_dataset(repo)+ if split_override:+ raw = load_dataset(repo, subset, split=split_override) if subset else load_dataset(repo, split=split_override)+ else:+ ds = load_dataset(repo, subset) if subset else load_dataset(repo)+ split = "train" if "train" in ds else list(ds.keys())[0]+ raw = ds[split]except Exception as e:print(f" ERROR: {e}")continue- split = "train" if "train" in ds else list(ds.keys())[0]- print(f" Converting to xlam format ({len(ds[split]):,} rows)...")- converted = CONVERTERS[name](ds[split])+ print(f" Converting to xlam format ({len(raw):,} rows)...")+ converted = CONVERTERS[name](raw)converted = converted.add_column("source", [name] * len(converted))before = len(converted)converted = converted.filter(lambda ex: (not _has_placeholder_args(ex["answers"])- and _answer_grounded(ex["query"], ex["answers"])+ and (ex["source"] in _VERIFIED_SOURCES+ or _answer_grounded(ex["query"], ex["answers"]))and _answer_calls_valid_tools(ex)and _answer_uses_valid_params(ex)),@@ -401,6 +748,8 @@ def load_and_combine():if dedup_dropped:print(f"\nDeduplicated: removed {dedup_dropped} duplicate queries")+ combined = _augment_irrelevance(combined, ratio=0.1, seed=42)+empty_count = sum(1 for ex in combined if json.loads(ex["answers"]) == [])print(f"\n{'='*60}")print(f"Combined dataset: {len(combined)} examples ({empty_count} with empty answers = no-call examples)")diff --git a/src/cli.py b/src/cli.pyindex 2a3b5d8..126ecbd 100644--- a/src/cli.py+++ b/src/cli.py@@ -16,8 +16,8 @@ def main():p = sub.add_parser("train", add_help=False)p.add_argument("--checkpoint", type=str, default=None)p.add_argument("--epochs", type=int, default=5)- p.add_argument("--batch-size", type=int, default=32)- p.add_argument("--lr", type=float, default=3e-4)+ p.add_argument("--batch-size", type=int, default=64)+ p.add_argument("--lr", type=float, default=4.5e-4)p.add_argument("--muon-lr", type=float, default=0.02)p.add_argument("--d-model", type=int, default=768)p.add_argument("--num-heads", type=int, default=16)@@ -43,13 +43,15 @@ def main():p.add_argument("--prune-end-frac", type=float, default=0.67,help="Fraction of epoch at which pruning finishes and mask locks (default: 0.67)")p.add_argument("--activation", type=str, default="drelu", choices=["drelu", "swiglu", "geglu"])- p.add_argument("--num-memory-slots", type=int, default=128)+ p.add_argument("--num-memory-slots", type=int, default=256)p.add_argument("--mat-factors", type=int, nargs="*", default=[2, 4],help="Matryoshka FFN shrink factors, e.g. 2=half width (default: 2 4)")- p.add_argument("--mat-shared-input", action="store_true",- help="Each unique input is repeated across all mat widths (default: unique input per width)")+ p.add_argument("--mat-shared-input", action=argparse.BooleanOptionalAction, default=True,+ help="Each unique input is repeated across all mat widths (default: True)")p.add_argument("--dropout", type=float, default=0.1,help="Dropout rate for residual connections (default: 0.1)")+ p.add_argument("--curriculum", action="store_true",+ help="Sort batches easy→hard by tool count each epoch")p = sub.add_parser("tokenize", add_help=False)p.add_argument("--max-samples", type=int, default=None,diff --git a/src/data.py b/src/data.pyindex e736eb0..12ab122 100644--- a/src/data.py+++ b/src/data.py@@ -126,6 +126,15 @@ def _mark_json_key_in_args(s, char_w, key, weight):char_w[m.start() + 1:m.start() + 1 + len(key)], weight)+def _count_tool_calls(answers_str):+ """Count the number of tool calls in an answers JSON string."""+ try:+ calls = _json.loads(answers_str)+ except (ValueError, TypeError):+ return 0+ return len(calls) if isinstance(calls, list) else 0++def _shuffle_tools_json(tools_str, seed):"""Parse tools JSON array, shuffle order deterministically, re-serialize."""try:@@ -432,22 +441,25 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_Decoder target: [<tool_call>, answer_tokens..., EOS, PAD...]Loss mask: token-level weights based on JSON structure (tool names, arg keys/values).- Returns (enc_inputs, dec_inputs, dec_targets, loss_mask, kept_indices).+ Returns (enc_inputs, dec_inputs, dec_targets, loss_mask, kept_indices, tool_counts)."""cache_id = _cache_key("toolcall", len(ds), max_enc_len, max_dec_len,w_name, w_value, w_key, shuffle_tools)cache_path = os.path.join(CACHE_DIR, cache_id)- tc_suffixes = ["_enc.npy", "_dec_in.npy", "_dec_tgt.npy", "_loss_mask.npy", "_kept_idx.npy"]+ tc_suffixes = ["_enc.npy", "_dec_in.npy", "_dec_tgt.npy", "_loss_mask.npy", "_kept_idx.npy", "_tool_count.npy"]def _load_tc_cache():+ tc_path = cache_path + "_tool_count.npy"+ tc = np.load(tc_path) if os.path.exists(tc_path) else Nonereturn (np.load(cache_path + "_enc.npy"),np.load(cache_path + "_dec_in.npy"),np.load(cache_path + "_dec_tgt.npy"),np.load(cache_path + "_loss_mask.npy"),np.load(cache_path + "_kept_idx.npy"),+ tc,)if os.path.exists(cache_path + "_enc.npy"):@@ -463,6 +475,8 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_tools_texts = [ex["tools"] for ex in ds]ans_texts = [ex["answers"] for ex in ds]+ tool_counts = np.array([_count_tool_calls(a) for a in ans_texts], dtype=np.int32)+if shuffle_tools:tools_texts = [_shuffle_tools_json(t, seed=i) for i, t in enumerate(tools_texts)]@@ -552,6 +566,7 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_dec_inputs = dec_inputs[keep]dec_targets = dec_targets[keep]loss_mask = loss_mask[keep]+ tool_counts = tool_counts[keep]print(f" Skipped {skipped} examples (too long for max_dec_len={max_dec_len})")else:kept_indices = np.arange(n, dtype=np.int64)@@ -561,14 +576,26 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_np.save(cache_path + "_dec_tgt.npy", dec_targets)np.save(cache_path + "_loss_mask.npy", loss_mask)np.save(cache_path + "_kept_idx.npy", kept_indices)+ np.save(cache_path + "_tool_count.npy", tool_counts)print(f"Cached {len(enc_inputs):,} tool-call pairs to {CACHE_DIR}/{cache_id}")- return enc_inputs, dec_inputs, dec_targets, loss_mask, kept_indices+ return enc_inputs, dec_inputs, dec_targets, loss_mask, kept_indices, tool_counts-def get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=True, loss_mask=None):+def get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=True, loss_mask=None, tool_counts=None):n = len(enc_inputs)- indices = np.random.permutation(n) if shuffle else np.arange(n)+ if tool_counts is not None:+ order = np.argsort(tool_counts, kind='stable')+ for c in np.unique(tool_counts):+ group = np.where(tool_counts[order] == c)[0]+ shuffled = order[group].copy()+ np.random.shuffle(shuffled)+ order[group] = shuffled+ indices = order+ elif shuffle:+ indices = np.random.permutation(n)+ else:+ indices = np.arange(n)for i in range(0, n - batch_size + 1, batch_size):idx = indices[i : i + batch_size]batch = (np.array(enc_inputs[idx]), np.array(dec_inputs[idx]), np.array(dec_targets[idx]))@@ -761,12 +788,15 @@ def load_prepared_data(split, mmap=False):)mmap_mode = "r" if mmap else None+ tc_path = cache_path + "_tool_count.npy"+ tc = np.load(tc_path, mmap_mode=mmap_mode) if os.path.exists(tc_path) else Nonereturn {"enc_inputs": np.load(cache_path + "_enc.npy", mmap_mode=mmap_mode),"dec_inputs": np.load(cache_path + "_dec_in.npy", mmap_mode=mmap_mode),"dec_targets": np.load(cache_path + "_dec_tgt.npy", mmap_mode=mmap_mode),"loss_mask": np.load(cache_path + "_loss_mask.npy", mmap_mode=mmap_mode),"kept_indices": np.load(cache_path + "_kept_idx.npy", mmap_mode=mmap_mode),+ "tool_counts": tc,}diff --git a/src/tokenize_data.py b/src/tokenize_data.pyindex 51f607e..a1d17cf 100644--- a/src/tokenize_data.py+++ b/src/tokenize_data.py@@ -57,7 +57,7 @@ def tokenize(args):w_value = getattr(args, "w_value", 2.0)w_key = getattr(args, "w_key", 1.5)shuffle_tools = getattr(args, "shuffle_tools", True)- _, _, _, _, kept_indices = prepare_tool_call_pairs(+ _, _, _, _, kept_indices, _ = prepare_tool_call_pairs(ds, tokenizer, max_enc_len=max_enc_len, max_dec_len=max_dec_len,w_name=w_name, w_value=w_value, w_key=w_key, shuffle_tools=shuffle_tools,)diff --git a/src/train.py b/src/train.pyindex c9e34a8..1b2001f 100644--- a/src/train.py+++ b/src/train.py@@ -392,6 +392,7 @@ def train(args):dec_inputs = train_data["dec_inputs"]dec_targets = train_data["dec_targets"]train_loss_mask = train_data["loss_mask"]+ tool_counts = train_data["tool_counts"]val_enc = val_data["enc_inputs"]val_dec_in = val_data["dec_inputs"]val_dec_tgt = val_data["dec_targets"]@@ -527,9 +528,11 @@ def train(args):epoch_step = 0text_losses = []+ _curriculum_tc = tool_counts if getattr(args, "curriculum", False) else Nonetext_batch_iter = PrefetchIterator(lambda: get_batches(enc_inputs, dec_inputs, dec_targets, unique_batch_size,- loss_mask=train_loss_mask),+ loss_mask=train_loss_mask,+ tool_counts=_curriculum_tc),prefetch=4,)