Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

tools dataset creation

HenryNdubuakue25f51d09bc0
diff --git a/scripts/tools_data.py b/scripts/tools_data.py
new file mode 100755
index 0000000..c1498b5
--- /dev/null
+++ b/scripts/tools_data.py
@@ -0,0 +1,290 @@
+"""
+Unified tool-calling dataset builder.
+
+Loads four HuggingFace tool-calling datasets and converts them all into the
+xlam-function-calling-60k schema:
+
+ query : str – the user's natural-language request
+ answers : str – JSON list of {"name": ..., "arguments": {...}} tool calls
+ tools : str – JSON list of available tool definitions
+ source : str – which dataset the example came from
+
+Datasets and their native formats:
+
+1. Salesforce/xlam-function-calling-60k
+ Already in the target schema (query, answers, tools).
+
+2. Salesforce/APIGen-MT-5k
+ Multi-turn conversations with human/gpt/function_call/observation roles.
+ We extract the first user message as the query and collect all function_call
+ messages (excluding internal "think" calls) as answers.
+
+3. glaiveai/glaive-function-calling-v2
+ Tool definitions embedded in the system prompt; chat is a flat string with
+ USER:/A: role markers and <functioncall> tags. ~50% of examples are
+ irrelevance cases (answers=[]) where the tools don't match the query — these
+ are kept intentionally so the model learns to decline rather than hallucinate.
+
+4. Team-ACE/ToolACE
+ Tool definitions as a JSON array in the system prompt; function calls use a
+ bracket syntax [FuncName(key=val, ...)] where names may contain spaces. Only
+ examples with at least one parsed function call are kept.
+"""
+
+import json
+import re
+from collections import Counter
+
+from datasets import load_dataset, Dataset, concatenate_datasets
+
+
+DATASETS = {
+ "xlam-function-calling-60k": "Salesforce/xlam-function-calling-60k",
+ "APIGen-MT-5k": "Salesforce/APIGen-MT-5k",
+ "glaive-function-calling": "glaiveai/glaive-function-calling-v2",
+ "ToolACE": "Team-ACE/ToolACE",
+}
+
+
+def convert_xlam(dataset):
+ return dataset.select_columns(["query", "answers", "tools"])
+
+
+def convert_apigen_mt(dataset):
+ rows = []
+ for ex in dataset:
+ convs = ex["conversations"]
+ tools = ex["tools"]
+
+ query = None
+ answers = []
+ for msg in convs:
+ if msg["from"] == "human" and query is None:
+ query = msg["value"]
+ elif msg["from"] == "function_call":
+ try:
+ fc = json.loads(msg["value"])
+ if fc.get("name") != "think":
+ answers.append(fc)
+ except json.JSONDecodeError:
+ pass
+
+ if query:
+ rows.append({
+ "query": query,
+ "answers": json.dumps(answers),
+ "tools": tools,
+ })
+ return Dataset.from_list(rows)
+
+
+def convert_glaive(dataset):
+ rows = []
+ for ex in dataset:
+ tools = _extract_json_objects(ex["system"])
+ query, answers = _parse_glaive_chat(ex["chat"])
+ if query:
+ rows.append({
+ "query": query,
+ "answers": json.dumps(answers),
+ "tools": json.dumps(tools),
+ })
+ return Dataset.from_list(rows)
+
+
+def _extract_json_objects(text):
+ """Extract top-level JSON objects with a 'name' field from text."""
+ objects = []
+ depth = 0
+ start = None
+ for i, ch in enumerate(text):
+ if ch == '{':
+ if depth == 0:
+ start = i
+ depth += 1
+ elif ch == '}':
+ depth -= 1
+ if depth == 0 and start is not None:
+ try:
+ obj = json.loads(text[start:i + 1])
+ if "name" in obj:
+ objects.append(obj)
+ except json.JSONDecodeError:
+ pass
+ start = None
+ return objects
+
+
+def _parse_glaive_chat(chat):
+ """Extract first user query and all function calls from glaive chat."""
+ query = None
+ answers = []
+
+ user_match = re.search(r'USER:\s*(.*?)(?=\n\s*(?:A:|ASSISTANT:)|$)', chat, re.DOTALL)
+ if user_match:
+ query = user_match.group(1).strip()
+
+ for m in re.finditer(r'<functioncall>\s*(\{.+?\})\s*(?:<\|endoftext\|>|$)', chat, re.DOTALL):
+ raw = m.group(1)
+ raw = re.sub(r"'(\{.*?\})'", r'"\1"', raw)
+ try:
+ fc = json.loads(raw)
+ except json.JSONDecodeError:
+ raw2 = re.sub(
+ r"'(\{.*?\})'",
+ lambda x: json.dumps(x.group(1)),
+ m.group(1),
+ )
+ try:
+ fc = json.loads(raw2)
+ except json.JSONDecodeError:
+ continue
+ args = fc.get("arguments", {})
+ if isinstance(args, str):
+ try:
+ args = json.loads(args)
+ except json.JSONDecodeError:
+ pass
+ answers.append({"name": fc["name"], "arguments": args})
+
+ return query, answers
+
+
+def convert_toolace(dataset):
+ rows = []
+ for ex in dataset:
+ tools = _parse_toolace_tools(ex["system"])
+ convs = ex["conversations"]
+
+ query = None
+ answers = []
+ for msg in convs:
+ if msg["from"] == "user" and query is None:
+ query = msg["value"]
+ elif msg["from"] == "assistant" and query is not None:
+ calls = _parse_toolace_calls(msg["value"])
+ if calls:
+ answers.extend(calls)
+ break
+
+ if query and answers:
+ rows.append({
+ "query": query,
+ "answers": json.dumps(answers),
+ "tools": json.dumps(tools),
+ })
+ return Dataset.from_list(rows)
+
+
+def _parse_toolace_tools(system_text):
+ """Extract tool list from ToolACE system prompt (JSON array after 'invoke:')."""
+ match = re.search(r'invoke:\s*(\[)', system_text)
+ if not match:
+ return []
+ start = match.start(1)
+ depth = 0
+ for i in range(start, len(system_text)):
+ if system_text[i] == '[':
+ depth += 1
+ elif system_text[i] == ']':
+ depth -= 1
+ if depth == 0:
+ try:
+ return json.loads(system_text[start:i + 1])
+ except json.JSONDecodeError:
+ return []
+ return []
+
+
+def _parse_toolace_calls(text):
+ """Parse [FuncName(key=val, ...)] calls. Function names may contain spaces."""
+ answers = []
+ bracket_match = re.search(r'\[(.+)\]', text, re.DOTALL)
+ if not bracket_match:
+ return answers
+
+ inner = bracket_match.group(1)
+ for m in re.finditer(r'([\w][\w ]*?\w|\w)\(([^)]*)\)', inner):
+ name = m.group(1).strip()
+ args_str = m.group(2)
+ arguments = {}
+ for arg_m in re.finditer(
+ r'(\w+)\s*=\s*(?:"((?:[^"\\]|\\.)*)"|\'((?:[^\'\\]|\\.)*)\'|(\[[^\]]*\])|([^,)]+))',
+ args_str,
+ ):
+ key = arg_m.group(1)
+ val = next(
+ (g for g in (arg_m.group(2), arg_m.group(3), arg_m.group(4), arg_m.group(5)) if g is not None),
+ None,
+ )
+ if val is not None:
+ val = val.strip()
+ try:
+ val = json.loads(val)
+ except (json.JSONDecodeError, TypeError):
+ pass
+ arguments[key] = val
+ answers.append({"name": name, "arguments": arguments})
+ return answers
+
+
+CONVERTERS = {
+ "xlam-function-calling-60k": convert_xlam,
+ "APIGen-MT-5k": convert_apigen_mt,
+ "glaive-function-calling": convert_glaive,
+ "ToolACE": convert_toolace,
+}
+
+
+def load_and_combine():
+ parts = []
+ for name, hf_id in DATASETS.items():
+ print(f"\nLoading {name} ({hf_id})...")
+ try:
+ ds = load_dataset(hf_id)
+ except Exception as e:
+ print(f" ERROR: {e}")
+ continue
+
+ print(f" Converting to xlam format...")
+ converted = CONVERTERS[name](ds["train"])
+ converted = converted.add_column("source", [name] * len(converted))
+ print(f" -> {len(converted)} examples")
+ parts.append(converted)
+
+ combined = concatenate_datasets(parts)
+ print(f"\n{'='*60}")
+ print(f"Combined dataset: {len(combined)} examples")
+ print(f"Columns: {combined.column_names}")
+
+ counts = Counter(combined["source"])
+ print(f"\nBreakdown:")
+ for src, cnt in counts.items():
+ print(f" {src}: {cnt}")
+
+ seen = set()
+ for i in range(len(combined)):
+ src = combined[i]["source"]
+ if src in seen:
+ continue
+ seen.add(src)
+ print(f"\n--- Sample from {src} ---")
+ for k in ["query", "answers", "tools"]:
+ v = str(combined[i][k])[:200]
+ print(f" {k}: {v}")
+
+ return combined
+
+
+if __name__ == "__main__":
+ combined = load_and_combine()
+ out_path = "data/tool_calls"
+ combined.save_to_disk(out_path)
+ print(f"\nSaved to {out_path}/")
+
+ from datasets import load_from_disk
+ reloaded = load_from_disk(out_path)
+ assert len(reloaded) == len(combined), f"Row count mismatch: {len(reloaded)} != {len(combined)}"
+ assert reloaded.column_names == combined.column_names, "Column mismatch"
+ assert reloaded[0] == combined[0], "First row mismatch"
+ print(f"Verified: {len(reloaded)} rows, columns {reloaded.column_names}")