Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: train tool-calling datasets with their tool list (#13180)

Nilaya3afe322fcfd
diff --git a/studio/backend/tests/test_chat_template_dataset_fallback.py b/studio/backend/tests/test_chat_template_dataset_fallback.py
index 0c59745a864..de83f8b5d12 100644
--- a/studio/backend/tests/test_chat_template_dataset_fallback.py
+++ b/studio/backend/tests/test_chat_template_dataset_fallback.py
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-"""Dataset template fallback and consistency across splits (#11321)."""
+"""dataset template fallback and consistency across splits (#11321)."""
from datasets import Dataset
@@ -11,6 +11,14 @@ from utils.datasets import chat_templates
OWN = "own-template"
OVERRIDE = "unsloth-override"
+_WEATHER = {
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "parameters": {"type": "object", "properties": {}},
+ },
+}
+
class _TemplatedTokenizer:
eos_token = "</s>"
@@ -27,6 +35,19 @@ class _TemplatedTokenizer:
return f"[{self.chat_template}] {turns}"
+class _OwnToolTemplateTokenizer(_TemplatedTokenizer):
+ def apply_chat_template(
+ self,
+ conversation,
+ tools = None,
+ **kwargs,
+ ):
+ rendered = super().apply_chat_template(conversation, **kwargs)
+ if self.chat_template == OWN and tools:
+ return f"[tools] {rendered}"
+ return rendered
+
+
def _plain_convo(index):
return [
{"role": "user", "content": f"question {index}"},
@@ -91,6 +112,34 @@ def test_override_is_kept_when_it_renders_the_dataset(monkeypatch):
assert tokenizer.chat_template == OVERRIDE
+def test_own_template_replaces_an_override_that_silently_drops_tools(monkeypatch):
+ tokenizer = _OwnToolTemplateTokenizer()
+ dataset_info = _dataset_info(_plain_convo)
+ dataset_info["dataset"] = dataset_info["dataset"].add_column(
+ "tools", [[_WEATHER] for _ in range(8)]
+ )
+
+ result = _format(dataset_info, tokenizer, monkeypatch)
+
+ assert result["success"] is True
+ assert all(text.startswith("[tools]") for text in result["dataset"]["text"])
+ assert tokenizer.chat_template == OWN
+
+
+def test_sparse_tool_catalog_is_included_in_the_template_probe(monkeypatch):
+ tokenizer = _OwnToolTemplateTokenizer()
+ dataset_info = _dataset_info(_plain_convo, rows = 40)
+ dataset_info["dataset"] = dataset_info["dataset"].add_column(
+ "tools", [[_WEATHER] if index == 37 else [] for index in range(40)]
+ )
+
+ result = _format(dataset_info, tokenizer, monkeypatch)
+
+ assert result["success"] is True
+ assert result["dataset"][37]["text"].startswith("[tools]")
+ assert tokenizer.chat_template == OWN
+
+
def test_override_is_kept_when_neither_template_renders(monkeypatch):
tokenizer = _TemplatedTokenizer()
broken = lambda index: [{"role": "broken", "content": f"row {index}"}]
diff --git a/studio/backend/tests/test_dataset_tools_column.py b/studio/backend/tests/test_dataset_tools_column.py
new file mode 100644
index 00000000000..38dc683559e
--- /dev/null
+++ b/studio/backend/tests/test_dataset_tools_column.py
@@ -0,0 +1,212 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import json
+
+import pytest
+from datasets import Dataset
+
+from utils.datasets import format_and_template_dataset
+
+_TEMPLATE = (
+ "{% if tools %}<tools>{% for tool in tools %}{{ tool | tojson }}{% endfor %}</tools>{% endif %}"
+ "{% for message in messages %}<{{ message.role }}>{{ message.content }}{% endfor %}"
+)
+
+
+class _ToolsTokenizer:
+ chat_template = _TEMPLATE
+ eos_token = "</s>"
+
+ def apply_chat_template(
+ self,
+ conversation,
+ tools = None,
+ **_kwargs,
+ ):
+ sandbox = pytest.importorskip("jinja2.sandbox")
+ env = sandbox.ImmutableSandboxedEnvironment()
+ env.filters["tojson"] = lambda value: json.dumps(value)
+ return env.from_string(self.chat_template).render(messages = conversation, tools = tools)
+
+
+class _NoToolsTokenizer(_ToolsTokenizer):
+ def apply_chat_template(
+ self,
+ conversation,
+ tools = None,
+ **kwargs,
+ ):
+ if tools is not None:
+ raise ValueError("tools are not supported")
+ return super().apply_chat_template(conversation, **kwargs)
+
+
+class _SystemWithToolsRejectedTokenizer(_ToolsTokenizer):
+ def apply_chat_template(
+ self,
+ conversation,
+ tools = None,
+ **kwargs,
+ ):
+ if tools is not None and conversation[0]["role"] == "system":
+ raise ValueError("system turns with tools are not supported")
+ return super().apply_chat_template(conversation, tools = tools, **kwargs)
+
+
+_WEATHER = {
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "description": "Current weather for a city",
+ "parameters": {"type": "object", "properties": {"city": {"type": "string"}}},
+ },
+}
+
+_SEARCH = {
+ "type": "function",
+ "function": {
+ "name": "web_search",
+ "description": "Search the web",
+ "parameters": {"type": "object", "properties": {"query": {"type": "string"}}},
+ },
+}
+
+_MESSAGES = [
+ {"role": "user", "content": "Weather in Paris?"},
+ {"role": "assistant", "content": "It is 21C."},
+]
+
+
+def _format(
+ rows,
+ tokenizer = None,
+ **kwargs,
+):
+ result = format_and_template_dataset(
+ Dataset.from_list(rows),
+ model_name = "stub-model",
+ tokenizer = tokenizer or _ToolsTokenizer(),
+ num_proc = 1,
+ **kwargs,
+ )
+ assert result["success"], result["errors"]
+ return list(result["dataset"]["text"])
+
+
+def _catalog(*tools):
+ return "<tools>" + "".join(json.dumps(tool) for tool in tools) + "</tools>"
+
+
+ "tools",
+ [[_WEATHER], json.dumps([_WEATHER])],
+ ids = ["list", "json_string"],
+)
+def test_tools_column_reaches_the_trained_text(tools):
+ texts = _format([{"messages": _MESSAGES, "tools": tools}])
+
+ assert texts == [_catalog(_WEATHER) + "<user>Weather in Paris?<assistant>It is 21C."]
+
+
+def test_flat_json_string_tools_are_trained_in_the_chat_tool_shape():
+ flat = json.dumps(_WEATHER["function"])
+
+ texts = _format([{"messages": _MESSAGES, "tools": [flat]}])
+
+ assert texts[0].startswith(_catalog(_WEATHER))
+
+
+def test_nested_tools_without_type_gain_the_function_discriminator():
+ nested = {"function": _WEATHER["function"]}
+
+ texts = _format([{"messages": _MESSAGES, "tools": [nested]}])
+
+ assert texts[0].startswith(_catalog({**nested, "type": "function"}))
+
+
+def test_each_row_trains_only_its_own_tools():
+ texts = _format(
+ [
+ {"messages": _MESSAGES, "tools": [_WEATHER]},
+ {"messages": _MESSAGES, "tools": [_SEARCH]},
+ {"messages": _MESSAGES, "tools": []},
+ ]
+ )
+
+ assert texts[0].startswith(_catalog(_WEATHER))
+ assert texts[1].startswith(_catalog(_SEARCH))
+ assert texts[2] == "<user>Weather in Paris?<assistant>It is 21C."
+
+
+def test_tools_and_system_column_are_both_trained():
+ texts = _format([{"system": "Be brief.", "messages": _MESSAGES, "tools": [_WEATHER]}])
+
+ assert texts == [
+ _catalog(_WEATHER) + "<system>Be brief.<user>Weather in Paris?<assistant>It is 21C."
+ ]
+
+
+def test_system_retry_keeps_the_tools_column_for_training():
+ texts = _format(
+ [{"system": "Be brief.", "messages": _MESSAGES, "tools": [_WEATHER]}],
+ _SystemWithToolsRejectedTokenizer(),
+ )
+
+ assert texts == [_catalog(_WEATHER) + "<user>Weather in Paris?<assistant>It is 21C."]
+
+
+def test_user_mapping_keeps_the_tools_column_for_training():
+ texts = _format(
+ [{"question": "Weather in Paris?", "answer": "It is 21C.", "tools": [_WEATHER]}],
+ custom_format_mapping = {"question": "user", "answer": "assistant"},
+ )
+
+ assert texts == [_catalog(_WEATHER) + "<user>Weather in Paris?<assistant>It is 21C."]
+
+
+ ("metadata", "system"),
+ [
+ ({"__system_prompt": "Be brief."}, "<system>Be brief."),
+ ({"__label_mapping": {}}, ""),
+ ],
+ ids = ["system_prompt", "label_mapping"],
+)
+def test_user_mapping_metadata_keeps_the_tools_column_for_training(metadata, system):
+ mapping = {"question": "user", "answer": "assistant", **metadata}
+
+ texts = _format(
+ [{"question": "Weather in Paris?", "answer": "It is 21C.", "tools": [_WEATHER]}],
+ custom_format_mapping = mapping,
+ )
+
+ assert texts == [_catalog(_WEATHER) + system + "<user>Weather in Paris?<assistant>It is 21C."]
+
+
+def test_rows_still_train_when_the_template_rejects_tools():
+ texts = _format([{"messages": _MESSAGES, "tools": [_WEATHER]}], _NoToolsTokenizer())
+
+ assert texts == ["<user>Weather in Paris?<assistant>It is 21C."]
+
+
+ "encode",
+ [json.dumps, lambda tools: [json.dumps(tool) for tool in tools]],
+ ids = ["json_string", "json_string_list"],
+)
+def test_json_string_tools_keep_their_null_defaults(encode):
+ tool = {
+ "type": "function",
+ "function": {
+ "name": "search_docs",
+ "parameters": {
+ "type": "object",
+ "properties": {"lang": {"type": "string", "default": None}},
+ },
+ },
+ }
+
+ texts = _format([{"messages": _MESSAGES, "tools": encode([tool])}])
+
+ assert texts[0].startswith(_catalog(tool))
diff --git a/studio/backend/utils/datasets/chat_templates.py b/studio/backend/utils/datasets/chat_templates.py
index 963a0b90f5a..f123d751623 100644
--- a/studio/backend/utils/datasets/chat_templates.py
+++ b/studio/backend/utils/datasets/chat_templates.py
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-"""Chat template utilities for dataset processing: apply chat templates to datasets and generate dataset info summaries."""
-
+import json
import warnings as python_warnings
from .cells import cell_text
@@ -164,8 +163,7 @@ def _set_chat_template(tokenizer, chat_template):
def _drop_none_values(value):
- # A loaded dict cannot tell an explicit null from a key another row added, so dict-typed
- # arguments lose their nulls; JSON-string arguments keep them.
+ # loaded dicts cannot distinguish nulls from keys added by another row, unlike JSON strings.
if isinstance(value, dict):
return {key: _drop_none_values(item) for key, item in value.items() if item is not None}
if isinstance(value, list):
@@ -173,7 +171,36 @@ def _drop_none_values(value):
return value
-def _render_conversation(tokenizer, conversation):
+def _json_cell(value):
+ if not isinstance(value, str):
+ return value
+ try:
+ return json.loads(value)
+ except ValueError:
+ return None
+
+
+def _row_tools(tools):
+ if isinstance(tools, list):
+ tools = [tool if isinstance(tool, str) else _drop_none_values(tool) for tool in tools]
+ else:
+ tools = _json_cell(tools)
+ if not isinstance(tools, list):
+ return None
+ tools = [_json_cell(tool) for tool in tools]
+ if not tools or not all(isinstance(tool, dict) for tool in tools):
+ return None
+ normalized = []
+ for tool in tools:
+ if tool.get("type") is None and isinstance(tool.get("function"), dict):
+ tool = {**tool, "type": "function"}
+ elif "function" not in tool and "name" in tool:
+ tool = {"type": "function", "function": tool}
+ normalized.append(tool)
+ return normalized
+
+
+def _render_conversation(tokenizer, conversation, tools = None, fallback_without_tools = True):
from core.inference.chat_template_helpers import _normalize_tool_call_arguments
attempts = []
@@ -181,35 +208,49 @@ def _render_conversation(tokenizer, conversation):
for attempt in (_normalize_tool_call_arguments(messages), messages):
if not any(attempt is seen for seen in attempts):
attempts.append(attempt)
+ tools_kwargs = {"tools": tools} if tools else {}
first_error = None
for attempt in attempts:
try:
return tokenizer.apply_chat_template(
- attempt, tokenize = False, add_generation_prompt = False
+ attempt, tokenize = False, add_generation_prompt = False, **tools_kwargs
)
except Exception as error:
- # The row as loaded is kept for templates that need a None content (DeepSeek V3), but its
- # error is usually a key the loader filled with None, so report the cleaned row's.
+ # allow DeepSeek V3 None content; prefer cleaned errors when loaders add None keys.
if first_error is None:
first_error = error
+ if tools and fallback_without_tools:
+ return _render_conversation(tokenizer, conversation)
raise first_error
-def _count_renderable(tokenizer, conversations):
+def _template_render_stats(tokenizer, rows):
rendered = 0
- for conversation in conversations:
+ advertised = 0
+ tool_rows = 0
+ for conversation, tools in rows:
try:
- _render_conversation(tokenizer, conversation)
+ if tools:
+ tool_rows += 1
+ with_tools = _render_conversation(tokenizer, conversation, tools)
+ try:
+ without_tools = _render_conversation(tokenizer, conversation)
+ except Exception:
+ advertised += 1
+ else:
+ advertised += with_tools != without_tools
+ else:
+ _render_conversation(tokenizer, conversation)
rendered += 1
except Exception:
pass
- return rendered
+ return rendered, advertised, tool_rows
-def _sample_conversations(dataset, chat_column, limit = _TEMPLATE_PROBE_ROWS):
- """Sample across the dataset, or from the start for streaming datasets."""
+def _sample_template_rows(dataset, chat_column, limit = _TEMPLATE_PROBE_ROWS):
+ """sample finite datasets evenly, adding one missed sparse tool row; stream from the front."""
n_rows = len(dataset) if hasattr(dataset, "__len__") else 0
- conversations = []
+ sampled = []
try:
if n_rows > limit:
step = (n_rows - 1) / (limit - 1)
@@ -219,42 +260,63 @@ def _sample_conversations(dataset, chat_column, limit = _TEMPLATE_PROBE_ROWS):
for row in rows:
conversation = row.get(chat_column)
if conversation:
- conversations.append(conversation)
- if len(conversations) >= limit:
+ sampled.append((conversation, _row_tools(row.get("tools"))))
+ if len(sampled) >= limit:
break
+ if (
+ n_rows > limit
+ and not any(tools for _, tools in sampled)
+ and "tools" in (getattr(dataset, "column_names", None) or ())
+ ):
+ for index, value in enumerate(dataset["tools"]):
+ tools = _row_tools(value)
+ if not tools:
+ continue
+ conversation = dataset[index].get(chat_column)
+ if conversation:
+ sampled.append((conversation, tools))
+ break
except Exception:
return []
- return conversations
+ return sampled
def keep_renderable_chat_template(tokenizer, dataset, chat_column, own_template):
- """Restore the checkpoint template if it renders more sampled rows; return a log note."""
+ """restore the checkpoint template when it renders more rows or preserves tool catalogs."""
override = getattr(tokenizer, "chat_template", None)
if not own_template or override == own_template:
return None
- conversations = _sample_conversations(dataset, chat_column)
- if not conversations:
+ sampled = _sample_template_rows(dataset, chat_column)
+ if not sampled:
return None
- rendered_by_override = _count_renderable(tokenizer, conversations)
- if rendered_by_override == len(conversations):
+ override_rendered, override_advertised, tool_rows = _template_render_stats(
+ tokenizer, sampled
+ )
+ if override_rendered == len(sampled) and override_advertised == tool_rows:
return None
_set_chat_template(tokenizer, own_template)
- if _count_renderable(tokenizer, conversations) <= rendered_by_override:
+ own_rendered, own_advertised, _ = _template_render_stats(tokenizer, sampled)
+ restores_tools = (
+ tool_rows
+ and override_advertised < tool_rows
+ and own_advertised == tool_rows
+ and own_rendered == len(sampled)
+ )
+ if own_rendered <= override_rendered and not restores_tools:
_set_chat_template(tokenizer, override)
return None
return (
- "📝 The Unsloth chat template cannot render this dataset's conversations "
- "(tool calls or consecutive same-role turns); using the model's own chat "
- "template instead"
+ "📝 The Unsloth chat template cannot render every conversation or tool catalog; "
+ "using the model's own chat template instead"
)
def resolve_dataset_chat_template(tokenizer, model_name, dataset, chat_column):
- """Choose on the first split and reuse for evaluation and saving."""
+ """choose a template on the first split and reuse it for evaluation and saving."""
remembered = getattr(tokenizer, _CHOSEN_TEMPLATE_ATTR, None)
if remembered is not None and remembered[0] == model_name:
_set_chat_template(tokenizer, remembered[1])
@@ -515,10 +577,7 @@ def apply_chat_template_to_dataset(
streamed_failures = []
- # Never clobber a real column: a dataset is allowed to already carry one named
- # like our marker, and remove_columns would then delete the user's own data.
- # A generator-backed IterableDataset reports column_names AND features as None,
- # so resolve_column_names' first-row probe is what sees the column there.
+ # protect real marker columns; generator-backed IterableDataset needs a first-row probe.
from .raw_text import resolve_column_names
existing_columns = set(resolve_column_names(dataset))
@@ -529,19 +588,26 @@ def apply_chat_template_to_dataset(
def _format_chatml(examples):
convos = examples[chat_column]
systems = examples.get("system") or [None] * len(convos)
+ row_tools = examples.get("tools") or [None] * len(convos)
texts = []
row_errors = []
- for convo, system in zip(convos, systems):
+ for convo, system, tools in zip(convos, systems, row_tools):
try:
with_system = _with_system_turn(convo, system)
+ tools = _row_tools(tools)
try:
- text = _render_conversation(tokenizer, with_system)
+ text = _render_conversation(
+ tokenizer,
+ with_system,
+ tools,
+ fallback_without_tools = with_system is convo,
+ )
except Exception:
- # A template without a system role still trains the conversation.
+ # unsupported system turns are omitted so the original conversation renders.
if with_system is convo:
raise
- text = _render_conversation(tokenizer, convo)
+ text = _render_conversation(tokenizer, convo, tools)
if remove_bos_prefix:
text = text.removeprefix('<bos>')
diff --git a/studio/backend/utils/datasets/dataset_utils.py b/studio/backend/utils/datasets/dataset_utils.py
index fd63d78b734..490881a049f 100644
--- a/studio/backend/utils/datasets/dataset_utils.py
+++ b/studio/backend/utils/datasets/dataset_utils.py
@@ -217,7 +217,10 @@ def _apply_user_mapping(
)
)
conversations.append(convo)
- return {"conversations": conversations}
+ mapped = {"conversations": conversations}
+ if "tools" in examples:
+ mapped["tools"] = examples["tools"]
+ return mapped
return dataset.map(
_convert,
@@ -289,7 +292,10 @@ def _apply_template_mapping(
convo.append({"role": "assistant", "content": "\n".join(asst_parts)})
conversations.append(convo)
- return {"conversations": conversations}
+ mapped = {"conversations": conversations}
+ if "tools" in examples:
+ mapped["tools"] = examples["tools"]
+ return mapped
return dataset.map(
_convert,
@@ -304,7 +310,11 @@ def _apply_user_mapping_alpaca(
mapping: dict,
batch_size: int = 1000,
):
- """Apply user-provided column mapping to convert dataset to Alpaca format. Accepts any format's role names, normalises via _TO_CHATML, then maps user -> instruction, system -> input, assistant -> output. Advisor ``__label_mapping`` names label values; ``__system_prompt`` is prepended to instruction, since system-role columns already fill input. Returns a dataset with instruction/input/output columns."""
+ """convert user mappings to Alpaca instruction, input, and output columns.
+
+ role names are normalized through ``_TO_CHATML`` and ``__label_mapping`` names label values.
+ ``__system_prompt`` prepends instruction because mapped system columns already fill input.
+ """
meta = {k: v for k, v in mapping.items() if k.startswith("__")}
column_roles = {k: v for k, v in mapping.items() if not k.startswith("__")}
system_prompt = meta.get("__system_prompt", "")