Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

curriculum training + irrelevant validation

HenryNdubuaku7732196ccaf0
diff --git a/scripts/tools_data.py b/scripts/tools_data.py
index da97119..bd8cd2e 100755
--- a/scripts/tools_data.py
+++ b/scripts/tools_data.py
@@ -22,9 +22,16 @@ Datasets:
Large-scale tool-calling dataset. Tool calls extracted from 'messages' column
(assistant messages with function_call objects), filtered by 'target_tools'.
Tool schemas converted from OpenAI format to flat format.
+
+4. Team-ACE/ToolACE
+ Synthetic tool-calling dataset (11.3k) with dual-layer verification.
+ Tool definitions in system prompt JSON; calls use bracket notation
+ [func(key=val)]. Grounding filter skipped (pre-verified).
"""
+import ast
import json
+import random
import re
from collections import Counter
@@ -37,6 +44,9 @@ DATASETS = {
"toucan-oss": ("Agent-Ark/Toucan-1.5M", "OSS"),
"toucan-kimi": ("Agent-Ark/Toucan-1.5M", "Kimi-K2"),
"toucan-qwen": ("Agent-Ark/Toucan-1.5M", "Qwen3"),
+ "toucan-sft": ("Agent-Ark/Toucan-1.5M", "SFT"),
+ "toolace": "Team-ACE/ToolACE",
+ "nemotron-tc": ("nvidia/Nemotron-Agentic-v1", None, "tool_calling"),
}
@@ -188,7 +198,7 @@ def convert_toucan(dataset):
query = (ex.get("question") or "").strip()
if not query:
continue
- tools = _convert_openai_tools(ex.get("available_tools", "[]"))
+ tools = _convert_openai_tools(ex.get("available_tools") or ex.get("tools") or "[]")
if not tools:
continue
tool_names = {t["name"] for t in tools}
@@ -203,12 +213,277 @@ def convert_toucan(dataset):
return Dataset.from_list(rows)
+def _extract_toolace_tools(system_text):
+ """Extract JSON tool definitions array from ToolACE system prompt."""
+ start = system_text.find('[{')
+ if start == -1:
+ return []
+ depth = 0
+ for i in range(start, len(system_text)):
+ if system_text[i] == '[':
+ depth += 1
+ elif system_text[i] == ']':
+ depth -= 1
+ if depth == 0:
+ try:
+ return json.loads(system_text[start:i + 1])
+ except json.JSONDecodeError:
+ return []
+ return []
+
+
+def _split_bracket_args(s, sep=','):
+ """Split on separator respecting nested brackets and quoted strings."""
+ parts = []
+ depth = 0
+ in_str = False
+ str_char = None
+ current = []
+ for i, ch in enumerate(s):
+ if in_str:
+ current.append(ch)
+ if ch == str_char and (i == 0 or s[i - 1] != '\\'):
+ in_str = False
+ elif ch in ('"', "'"):
+ in_str = True
+ str_char = ch
+ current.append(ch)
+ elif ch in ('(', '[', '{'):
+ depth += 1
+ current.append(ch)
+ elif ch in (')', ']', '}'):
+ depth -= 1
+ current.append(ch)
+ elif ch == sep and depth == 0:
+ parts.append(''.join(current))
+ current = []
+ else:
+ current.append(ch)
+ if current:
+ parts.append(''.join(current))
+ return parts
+
+
+def _parse_bracket_value(s):
+ """Parse a value string from bracket notation arguments."""
+ s = s.strip()
+ if not s:
+ return s
+ try:
+ return json.loads(s)
+ except (json.JSONDecodeError, ValueError):
+ pass
+ if s == 'True':
+ return True
+ if s == 'False':
+ return False
+ if s in ('None', 'null'):
+ return None
+ try:
+ return ast.literal_eval(s)
+ except (ValueError, SyntaxError):
+ pass
+ return s
+
+
+def _parse_toolace_calls(text, tool_names):
+ """Parse ToolACE bracket notation into tool calls using known names."""
+ text = text.strip()
+ if not text.startswith('[') or not text.endswith(']'):
+ return []
+ inner = text[1:-1].strip()
+ if not inner:
+ return []
+
+ sorted_names = sorted(tool_names, key=len, reverse=True)
+ calls = []
+ pos = 0
+
+ while pos < len(inner):
+ while pos < len(inner) and inner[pos] in (' ', ',', '\n', '\t'):
+ pos += 1
+ if pos >= len(inner):
+ break
+
+ matched = None
+ for name in sorted_names:
+ if inner[pos:pos + len(name)] == name:
+ rest = inner[pos + len(name):].lstrip()
+ if rest and rest[0] == '(':
+ matched = name
+ break
+
+ if not matched:
+ break
+
+ pos += len(matched)
+ while pos < len(inner) and inner[pos] == ' ':
+ pos += 1
+ if pos >= len(inner) or inner[pos] != '(':
+ break
+
+ depth = 0
+ in_str = False
+ str_char = None
+ end = pos
+ while end < len(inner):
+ ch = inner[end]
+ if in_str:
+ if ch == str_char and (end == 0 or inner[end - 1] != '\\'):
+ in_str = False
+ elif ch in ('"', "'"):
+ in_str = True
+ str_char = ch
+ elif ch == '(':
+ depth += 1
+ elif ch == ')':
+ depth -= 1
+ if depth == 0:
+ break
+ end += 1
+
+ args_str = inner[pos + 1:end].strip()
+ arguments = {}
+ if args_str:
+ for pair in _split_bracket_args(args_str, ','):
+ pair = pair.strip()
+ eq = pair.find('=')
+ if eq < 0:
+ continue
+ key = pair[:eq].strip()
+ val = _parse_bracket_value(pair[eq + 1:])
+ arguments[key] = val
+
+ calls.append({"name": matched, "arguments": arguments})
+ pos = end + 1
+
+ return calls
+
+
+def convert_toolace(dataset):
+ rows = []
+ for ex in dataset:
+ system = ex.get("system", "")
+ convs = ex.get("conversations", [])
+
+ raw_tools = _extract_toolace_tools(system)
+ tools = _convert_openai_tools(raw_tools)
+ if not tools:
+ continue
+
+ tool_names = {t["name"] for t in tools}
+
+ query = None
+ assistant_text = None
+ for msg in convs:
+ role = msg.get("from", "")
+ val = (msg.get("value") or "").strip()
+ if role == "user" and query is None:
+ query = val
+ elif role == "assistant" and assistant_text is None:
+ assistant_text = val
+
+ if not query:
+ continue
+
+ calls = []
+ if assistant_text:
+ calls = _parse_toolace_calls(assistant_text, tool_names)
+
+ rows.append({
+ "query": query,
+ "answers": json.dumps(calls),
+ "tools": json.dumps(tools),
+ })
+ return Dataset.from_list(rows)
+
+
+def _extract_nemotron_calls(messages, tool_names=None):
+ """Extract tool calls from Nemotron messages (OpenAI tool_calls format)."""
+ calls = []
+ for msg in messages:
+ if not isinstance(msg, dict) or msg.get("role") != "assistant":
+ continue
+
+ # OpenAI tool_calls format (newer)
+ for tc in (msg.get("tool_calls") or []):
+ if not isinstance(tc, dict):
+ continue
+ fn = tc.get("function", tc)
+ name = fn.get("name", "")
+ if not name:
+ continue
+ if tool_names and name not in tool_names:
+ continue
+ args = fn.get("arguments", {})
+ if isinstance(args, str):
+ try:
+ args = json.loads(args)
+ except (json.JSONDecodeError, TypeError):
+ args = {}
+ if not isinstance(args, dict):
+ args = {}
+ calls.append({"name": name, "arguments": args})
+
+ # Fallback: function_call format (older OpenAI)
+ fc = msg.get("function_call")
+ if fc and isinstance(fc, dict):
+ name = fc.get("name", "")
+ if name and (not tool_names or name in tool_names):
+ args = fc.get("arguments", {})
+ if isinstance(args, str):
+ try:
+ args = json.loads(args)
+ except (json.JSONDecodeError, TypeError):
+ args = {}
+ if not isinstance(args, dict):
+ args = {}
+ calls.append({"name": name, "arguments": args})
+
+ return calls
+
+
+def convert_nemotron(dataset):
+ rows = []
+ for ex in dataset:
+ tools_raw = ex.get("tools") or []
+ tools = _convert_openai_tools(tools_raw)
+ if not tools:
+ continue
+
+ tool_names = {t["name"] for t in tools}
+ messages = ex.get("messages") or []
+
+ query = None
+ for msg in messages:
+ if isinstance(msg, dict) and msg.get("role") == "user":
+ query = (msg.get("content") or "").strip()
+ break
+
+ if not query:
+ continue
+
+ calls = _extract_nemotron_calls(messages, tool_names)
+ if not calls:
+ continue
+
+ rows.append({
+ "query": query,
+ "answers": json.dumps(calls),
+ "tools": json.dumps(tools),
+ })
+ return Dataset.from_list(rows)
+
+
CONVERTERS = {
"xlam-function-calling-60k": convert_xlam,
"glaive-function-calling": convert_glaive,
"toucan-oss": convert_toucan,
"toucan-kimi": convert_toucan,
"toucan-qwen": convert_toucan,
+ "toucan-sft": convert_toucan,
+ "toolace": convert_toolace,
+ "nemotron-tc": convert_nemotron,
}
_PLACEHOLDER_VALUES = {
@@ -331,6 +606,10 @@ def _answer_uses_valid_params(ex):
return True
+# Sources with their own verification — skip grounding check
+_VERIFIED_SOURCES = {"toolace"}
+
+
def _deduplicate(dataset):
"""Remove duplicate queries, keeping the first occurrence."""
seen = set()
@@ -345,32 +624,100 @@ def _deduplicate(dataset):
return dataset.select(keep)
+def _augment_irrelevance(dataset, ratio=0.1, seed=42):
+ """Create irrelevance samples by swapping tools with unrelated queries.
+
+ For a fraction of examples with tool calls, replace the available tools
+ with tools from a different example and set the answer to [] (no calls).
+ Ensures no overlap between the query's needed tools and the donor tools.
+ """
+ rng = random.Random(seed)
+
+ all_answers = dataset["answers"]
+ all_tools = dataset["tools"]
+ all_queries = dataset["query"]
+
+ with_calls = []
+ call_names = {}
+ tool_names_by_idx = {}
+
+ for i, ans_str in enumerate(all_answers):
+ try:
+ answers = json.loads(ans_str)
+ except (json.JSONDecodeError, TypeError):
+ continue
+ if not answers:
+ continue
+ with_calls.append(i)
+ call_names[i] = {c["name"] for c in answers if isinstance(c, dict)}
+ try:
+ tools = json.loads(all_tools[i])
+ tool_names_by_idx[i] = {t["name"] for t in tools if isinstance(t, dict)}
+ except (json.JSONDecodeError, TypeError):
+ tool_names_by_idx[i] = set()
+
+ n_aug = int(len(with_calls) * ratio)
+ selected = rng.sample(with_calls, min(n_aug, len(with_calls)))
+
+ rows = []
+ for idx in selected:
+ orig_call_names = call_names[idx]
+ for _ in range(5):
+ donor = rng.choice(with_calls)
+ if donor == idx:
+ continue
+ # Ensure none of the query's needed tools appear in donor's tool set
+ if not (orig_call_names & tool_names_by_idx.get(donor, set())):
+ rows.append({
+ "query": all_queries[idx],
+ "answers": "[]",
+ "tools": all_tools[donor],
+ "source": "irrelevance-aug",
+ })
+ break
+
+ if rows:
+ print(f"\nIrrelevance augmentation: {len(rows)} samples "
+ f"({len(rows)/len(dataset)*100:.1f}% of dataset)")
+ return concatenate_datasets([dataset, Dataset.from_list(rows)])
+ return dataset
+
+
def load_and_combine():
parts = []
for name, hf_id in DATASETS.items():
+ split_override = None
if isinstance(hf_id, tuple):
- repo, subset = hf_id
- label = f"{repo} [{subset}]"
+ if len(hf_id) == 3:
+ repo, subset, split_override = hf_id
+ else:
+ repo, subset = hf_id
+ label = f"{repo}" + (f" [{subset}]" if subset else "")
else:
repo, subset = hf_id, None
label = repo
print(f"\nLoading {name} ({label})...")
try:
- ds = load_dataset(repo, subset) if subset else load_dataset(repo)
+ if split_override:
+ raw = load_dataset(repo, subset, split=split_override) if subset else load_dataset(repo, split=split_override)
+ else:
+ ds = load_dataset(repo, subset) if subset else load_dataset(repo)
+ split = "train" if "train" in ds else list(ds.keys())[0]
+ raw = ds[split]
except Exception as e:
print(f" ERROR: {e}")
continue
- split = "train" if "train" in ds else list(ds.keys())[0]
- print(f" Converting to xlam format ({len(ds[split]):,} rows)...")
- converted = CONVERTERS[name](ds[split])
+ print(f" Converting to xlam format ({len(raw):,} rows)...")
+ converted = CONVERTERS[name](raw)
converted = converted.add_column("source", [name] * len(converted))
before = len(converted)
converted = converted.filter(
lambda ex: (
not _has_placeholder_args(ex["answers"])
- and _answer_grounded(ex["query"], ex["answers"])
+ and (ex["source"] in _VERIFIED_SOURCES
+ or _answer_grounded(ex["query"], ex["answers"]))
and _answer_calls_valid_tools(ex)
and _answer_uses_valid_params(ex)
),
@@ -401,6 +748,8 @@ def load_and_combine():
if dedup_dropped:
print(f"\nDeduplicated: removed {dedup_dropped} duplicate queries")
+ combined = _augment_irrelevance(combined, ratio=0.1, seed=42)
+
empty_count = sum(1 for ex in combined if json.loads(ex["answers"]) == [])
print(f"\n{'='*60}")
print(f"Combined dataset: {len(combined)} examples ({empty_count} with empty answers = no-call examples)")
diff --git a/src/cli.py b/src/cli.py
index 2a3b5d8..126ecbd 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -16,8 +16,8 @@ def main():
p = sub.add_parser("train", add_help=False)
p.add_argument("--checkpoint", type=str, default=None)
p.add_argument("--epochs", type=int, default=5)
- p.add_argument("--batch-size", type=int, default=32)
- p.add_argument("--lr", type=float, default=3e-4)
+ p.add_argument("--batch-size", type=int, default=64)
+ p.add_argument("--lr", type=float, default=4.5e-4)
p.add_argument("--muon-lr", type=float, default=0.02)
p.add_argument("--d-model", type=int, default=768)
p.add_argument("--num-heads", type=int, default=16)
@@ -43,13 +43,15 @@ def main():
p.add_argument("--prune-end-frac", type=float, default=0.67,
help="Fraction of epoch at which pruning finishes and mask locks (default: 0.67)")
p.add_argument("--activation", type=str, default="drelu", choices=["drelu", "swiglu", "geglu"])
- p.add_argument("--num-memory-slots", type=int, default=128)
+ p.add_argument("--num-memory-slots", type=int, default=256)
p.add_argument("--mat-factors", type=int, nargs="*", default=[2, 4],
help="Matryoshka FFN shrink factors, e.g. 2=half width (default: 2 4)")
- p.add_argument("--mat-shared-input", action="store_true",
- help="Each unique input is repeated across all mat widths (default: unique input per width)")
+ p.add_argument("--mat-shared-input", action=argparse.BooleanOptionalAction, default=True,
+ help="Each unique input is repeated across all mat widths (default: True)")
p.add_argument("--dropout", type=float, default=0.1,
help="Dropout rate for residual connections (default: 0.1)")
+ p.add_argument("--curriculum", action="store_true",
+ help="Sort batches easy→hard by tool count each epoch")
p = sub.add_parser("tokenize", add_help=False)
p.add_argument("--max-samples", type=int, default=None,
diff --git a/src/data.py b/src/data.py
index e736eb0..12ab122 100644
--- a/src/data.py
+++ b/src/data.py
@@ -126,6 +126,15 @@ def _mark_json_key_in_args(s, char_w, key, weight):
char_w[m.start() + 1:m.start() + 1 + len(key)], weight)
+def _count_tool_calls(answers_str):
+ """Count the number of tool calls in an answers JSON string."""
+ try:
+ calls = _json.loads(answers_str)
+ except (ValueError, TypeError):
+ return 0
+ return len(calls) if isinstance(calls, list) else 0
+
+
def _shuffle_tools_json(tools_str, seed):
"""Parse tools JSON array, shuffle order deterministically, re-serialize."""
try:
@@ -432,22 +441,25 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_
Decoder target: [<tool_call>, answer_tokens..., EOS, PAD...]
Loss mask: token-level weights based on JSON structure (tool names, arg keys/values).
- Returns (enc_inputs, dec_inputs, dec_targets, loss_mask, kept_indices).
+ Returns (enc_inputs, dec_inputs, dec_targets, loss_mask, kept_indices, tool_counts).
"""
cache_id = _cache_key("toolcall", len(ds), max_enc_len, max_dec_len,
w_name, w_value, w_key, shuffle_tools)
cache_path = os.path.join(CACHE_DIR, cache_id)
- tc_suffixes = ["_enc.npy", "_dec_in.npy", "_dec_tgt.npy", "_loss_mask.npy", "_kept_idx.npy"]
+ tc_suffixes = ["_enc.npy", "_dec_in.npy", "_dec_tgt.npy", "_loss_mask.npy", "_kept_idx.npy", "_tool_count.npy"]
def _load_tc_cache():
+ tc_path = cache_path + "_tool_count.npy"
+ tc = np.load(tc_path) if os.path.exists(tc_path) else None
return (
np.load(cache_path + "_enc.npy"),
np.load(cache_path + "_dec_in.npy"),
np.load(cache_path + "_dec_tgt.npy"),
np.load(cache_path + "_loss_mask.npy"),
np.load(cache_path + "_kept_idx.npy"),
+ tc,
)
if os.path.exists(cache_path + "_enc.npy"):
@@ -463,6 +475,8 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_
tools_texts = [ex["tools"] for ex in ds]
ans_texts = [ex["answers"] for ex in ds]
+ tool_counts = np.array([_count_tool_calls(a) for a in ans_texts], dtype=np.int32)
+
if shuffle_tools:
tools_texts = [_shuffle_tools_json(t, seed=i) for i, t in enumerate(tools_texts)]
@@ -552,6 +566,7 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_
dec_inputs = dec_inputs[keep]
dec_targets = dec_targets[keep]
loss_mask = loss_mask[keep]
+ tool_counts = tool_counts[keep]
print(f" Skipped {skipped} examples (too long for max_dec_len={max_dec_len})")
else:
kept_indices = np.arange(n, dtype=np.int64)
@@ -561,14 +576,26 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_
np.save(cache_path + "_dec_tgt.npy", dec_targets)
np.save(cache_path + "_loss_mask.npy", loss_mask)
np.save(cache_path + "_kept_idx.npy", kept_indices)
+ np.save(cache_path + "_tool_count.npy", tool_counts)
print(f"Cached {len(enc_inputs):,} tool-call pairs to {CACHE_DIR}/{cache_id}")
- return enc_inputs, dec_inputs, dec_targets, loss_mask, kept_indices
+ return enc_inputs, dec_inputs, dec_targets, loss_mask, kept_indices, tool_counts
-def get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=True, loss_mask=None):
+def get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=True, loss_mask=None, tool_counts=None):
n = len(enc_inputs)
- indices = np.random.permutation(n) if shuffle else np.arange(n)
+ if tool_counts is not None:
+ order = np.argsort(tool_counts, kind='stable')
+ for c in np.unique(tool_counts):
+ group = np.where(tool_counts[order] == c)[0]
+ shuffled = order[group].copy()
+ np.random.shuffle(shuffled)
+ order[group] = shuffled
+ indices = order
+ elif shuffle:
+ indices = np.random.permutation(n)
+ else:
+ indices = np.arange(n)
for i in range(0, n - batch_size + 1, batch_size):
idx = indices[i : i + batch_size]
batch = (np.array(enc_inputs[idx]), np.array(dec_inputs[idx]), np.array(dec_targets[idx]))
@@ -761,12 +788,15 @@ def load_prepared_data(split, mmap=False):
)
mmap_mode = "r" if mmap else None
+ tc_path = cache_path + "_tool_count.npy"
+ tc = np.load(tc_path, mmap_mode=mmap_mode) if os.path.exists(tc_path) else None
return {
"enc_inputs": np.load(cache_path + "_enc.npy", mmap_mode=mmap_mode),
"dec_inputs": np.load(cache_path + "_dec_in.npy", mmap_mode=mmap_mode),
"dec_targets": np.load(cache_path + "_dec_tgt.npy", mmap_mode=mmap_mode),
"loss_mask": np.load(cache_path + "_loss_mask.npy", mmap_mode=mmap_mode),
"kept_indices": np.load(cache_path + "_kept_idx.npy", mmap_mode=mmap_mode),
+ "tool_counts": tc,
}
diff --git a/src/tokenize_data.py b/src/tokenize_data.py
index 51f607e..a1d17cf 100644
--- a/src/tokenize_data.py
+++ b/src/tokenize_data.py
@@ -57,7 +57,7 @@ def tokenize(args):
w_value = getattr(args, "w_value", 2.0)
w_key = getattr(args, "w_key", 1.5)
shuffle_tools = getattr(args, "shuffle_tools", True)
- _, _, _, _, kept_indices = prepare_tool_call_pairs(
+ _, _, _, _, kept_indices, _ = prepare_tool_call_pairs(
ds, tokenizer, max_enc_len=max_enc_len, max_dec_len=max_dec_len,
w_name=w_name, w_value=w_value, w_key=w_key, shuffle_tools=shuffle_tools,
)
diff --git a/src/train.py b/src/train.py
index c9e34a8..1b2001f 100644
--- a/src/train.py
+++ b/src/train.py
@@ -392,6 +392,7 @@ def train(args):
dec_inputs = train_data["dec_inputs"]
dec_targets = train_data["dec_targets"]
train_loss_mask = train_data["loss_mask"]
+ tool_counts = train_data["tool_counts"]
val_enc = val_data["enc_inputs"]
val_dec_in = val_data["dec_inputs"]
val_dec_tgt = val_data["dec_targets"]
@@ -527,9 +528,11 @@ def train(args):
epoch_step = 0
text_losses = []
+ _curriculum_tc = tool_counts if getattr(args, "curriculum", False) else None
text_batch_iter = PrefetchIterator(
lambda: get_batches(enc_inputs, dec_inputs, dec_targets, unique_batch_size,
- loss_mask=train_loss_mask),
+ loss_mask=train_loss_mask,
+ tool_counts=_curriculum_tc),
prefetch=4,
)