Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep tool calls and results when training on ShareGPT tool data (#13181)

Nilay0bdb7d5967c5
diff --git a/studio/backend/tests/test_chat_template_tool_call_rows.py b/studio/backend/tests/test_chat_template_tool_call_rows.py
index 67805cf75da..e4b54a04b3c 100644
--- a/studio/backend/tests/test_chat_template_tool_call_rows.py
+++ b/studio/backend/tests/test_chat_template_tool_call_rows.py
@@ -7,7 +7,7 @@ from pathlib import Path
import pytest
from datasets import Dataset
-from utils.datasets import apply_chat_template_to_dataset
+from utils.datasets import apply_chat_template_to_dataset, format_and_template_dataset
from utils.datasets.chat_templates import keep_renderable_chat_template
_GEMMA4_TEMPLATE = (
@@ -221,3 +221,309 @@ def test_template_probe_counts_rows_after_cleaning():
assert note is None
assert tokenizer.chat_template == _LLAMA3_TEMPLATE
+
+
+def _sharegpt_tool_row(call):
+ return {
+ "conversations": [
+ {"from": "human", "value": "Weather in Paris?"},
+ {"from": "function_call", "value": call},
+ {"from": "observation", "value": '{"temp": 18}'},
+ {"from": "gpt", "value": "It is 18C in Paris."},
+ ]
+ }
+
+
+def _format_sharegpt(rows, template):
+ return format_and_template_dataset(
+ Dataset.from_list(rows),
+ model_name = "stub-model",
+ tokenizer = _JinjaTokenizer(template),
+ num_proc = 1,
+ )
+
+
+def test_sharegpt_function_call_and_observation_train_as_tool_turns():
+ call = json.dumps({"name": "get_weather", "arguments": {"city": "Paris"}})
+
+ result = _format_sharegpt([_sharegpt_tool_row(call)], _LLAMA3_TEMPLATE)
+
+ assert result["success"] is True, result["errors"]
+ text = result["dataset"][0]["text"]
+ assert (
+ "<|start_header_id|>assistant<|end_header_id|>\n\n"
+ '{"name": "get_weather", "parameters": {"city": "Paris"}}' in text
+ )
+ assert '<|start_header_id|>ipython<|end_header_id|>\n\n{"temp": 18}' in text
+ assert "function_call" not in text
+ assert "observation" not in text
+
+
+def test_sharegpt_function_call_list_trains_every_call():
+ call = json.dumps(
+ [
+ {"name": "get_weather", "arguments": {"city": "Paris"}},
+ {"name": "get_weather", "arguments": '{"city": "Rome"}'},
+ ]
+ )
+
+ result = _format_sharegpt([_sharegpt_tool_row(call)], _QWEN35_TEMPLATE)
+
+ assert result["success"] is True, result["errors"]
+ text = result["dataset"][0]["text"]
+ assert "<|im_start|>assistant\n<tool_call>\n<function=get_weather>" in text
+ assert "<parameter=city>\nParis\n</parameter>" in text
+ assert "<parameter=city>\nRome\n</parameter>" in text
+ assert '<|im_start|>tool\n{"temp": 18}' in text
+
+
+def test_sharegpt_function_call_that_is_not_json_is_kept_as_written():
+ result = _format_sharegpt([_sharegpt_tool_row("get_weather(Paris)")], _QWEN35_TEMPLATE)
+
+ assert result["success"] is True, result["errors"]
+ assert "<|im_start|>function_call\nget_weather(Paris)" in result["dataset"][0]["text"]
+
+
+def test_sharegpt_function_call_is_kept_when_the_template_ignores_tool_calls():
+ plain_chatml = (
+ "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\\n'"
+ " + message['content'] + '<|im_end|>\\n'}}{% endfor %}"
+ )
+ call = json.dumps({"name": "get_weather", "arguments": {"city": "Paris"}})
+
+ result = _format_sharegpt([_sharegpt_tool_row(call)], plain_chatml)
+
+ assert result["success"] is True, result["errors"]
+ assert f"<|im_start|>function_call\n{call}" in result["dataset"][0]["text"]
+
+
+def test_sharegpt_function_call_renders_on_a_template_gating_calls_on_null_content():
+ call = json.dumps({"name": "get_weather", "arguments": {"city": "Paris"}})
+
+ result = _format_sharegpt([_sharegpt_tool_row(call)], _DEEPSEEK_TEMPLATE)
+
+ assert result["success"] is True, result["errors"]
+ text = result["dataset"][0]["text"]
+ assert '<call>get_weather\n{"city": "Paris"}</call>' in text
+ assert '<output>{"temp": 18}' in text
+
+
+def test_sharegpt_call_name_in_the_prompt_does_not_hide_an_ignored_call():
+ plain_chatml = (
+ "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\\n'"
+ " + message['content'] + '<|im_end|>\\n'}}{% endfor %}"
+ )
+ call = json.dumps({"name": "get_weather", "arguments": {"city": "Paris"}})
+ row = _sharegpt_tool_row(call)
+ row["conversations"][0]["value"] = "Use get_weather for Paris."
+
+ result = _format_sharegpt([row], plain_chatml)
+
+ assert f"<|im_start|>function_call\n{call}" in result["dataset"][0]["text"]
+
+
+def test_sharegpt_function_call_keeps_explicit_null_arguments():
+ call = json.dumps({"name": "get_weather", "arguments": {"city": "Paris", "unit": None}})
+
+ result = _format_sharegpt([_sharegpt_tool_row(call)], _LLAMA3_TEMPLATE)
+
+ assert result["success"] is True, result["errors"]
+ assert '"parameters": {"city": "Paris", "unit": null}' in result["dataset"][0]["text"]
+
+
+def test_sharegpt_parallel_calls_are_split_for_one_call_templates():
+ call = json.dumps(
+ [
+ {"name": "get_weather", "arguments": {"city": "Paris"}},
+ {"name": "get_time", "arguments": {"city": "Rome"}},
+ ]
+ )
+
+ row = _sharegpt_tool_row(call)
+ row["conversations"].insert(3, {"from": "observation", "value": '{"time": "noon"}'})
+
+ result = _format_sharegpt([row], _LLAMA3_TEMPLATE)
+
+ assert result["success"] is True, result["errors"]
+ text = result["dataset"][0]["text"]
+ paris = text.index('{"name": "get_weather", "parameters": {"city": "Paris"}}')
+ rome = text.index('{"name": "get_time", "parameters": {"city": "Rome"}}')
+ assert paris < text.index('{"temp": 18}') < rome < text.index('{"time": "noon"}')
+ assert "function_call" not in text
+
+
+def test_sharegpt_parallel_calls_sharing_one_result_render_natively_in_order():
+ call = json.dumps(
+ [
+ {"name": "get_weather", "arguments": {"city": "Paris"}},
+ {"name": "get_time", "arguments": {"city": "Rome"}},
+ ]
+ )
+
+ result = _format_sharegpt([_sharegpt_tool_row(call)], _LLAMA3_TEMPLATE)
+
+ assert result["success"] is True, result["errors"]
+ text = result["dataset"][0]["text"]
+ assert text.index("Rome") < text.index('{"temp": 18}')
+ assert "function_call" not in text
+
+
+def test_sharegpt_repeated_call_name_is_not_lost_on_a_first_call_only_template():
+ first_call_only = (
+ "{%- for message in messages %}"
+ "{%- if message.tool_calls %}{%- set call = message.tool_calls[0].function %}"
+ "{{- '<call ' + call.name + '>' + call.arguments + '</call>' }}"
+ "{%- elif message.role == 'tool' %}{{- '<result from=get_weather>' + message.content }}"
+ "{%- else %}{{- '<' + message.role + '>' + message.content }}{%- endif %}"
+ "{%- endfor %}"
+ )
+ call = json.dumps(
+ [
+ {"name": "get_weather", "arguments": {"city": "Paris"}},
+ {"name": "get_weather", "arguments": {"city": "Rome"}},
+ ]
+ )
+ row = _sharegpt_tool_row(call)
+ row["conversations"].insert(3, {"from": "observation", "value": '{"temp": 21}'})
+
+ result = _format_sharegpt([row], first_call_only)
+
+ assert result["success"] is True, result["errors"]
+ assert "Rome" in result["dataset"][0]["text"]
+
+
+def test_sharegpt_results_are_kept_when_the_template_drops_tool_turns():
+ no_tool_turns = (
+ "{%- for message in messages %}"
+ "{%- if message.tool_calls %}{%- set call = message.tool_calls[0].function %}"
+ "{{- '<call>' + call.name + call.arguments }}"
+ "{%- elif message.role != 'tool' %}{{- '<' + message.role + '>' + message.content }}"
+ "{%- endif %}{%- endfor %}"
+ )
+ call = json.dumps({"name": "get_weather", "arguments": {"city": "Paris"}})
+
+ result = _format_sharegpt([_sharegpt_tool_row(call)], no_tool_turns)
+
+ assert '{"temp": 18}' in result["dataset"][0]["text"]
+
+
+def test_sharegpt_parallel_results_are_labelled_with_their_own_call():
+ call = json.dumps(
+ [
+ {"name": "get_weather", "arguments": {"city": "Paris"}},
+ {"name": "get_time", "arguments": {"city": "Rome"}},
+ ]
+ )
+ row = _sharegpt_tool_row(call)
+ row["conversations"].insert(3, {"from": "observation", "value": '{"time": "noon"}'})
+
+ result = _format_sharegpt([row], _GEMMA4_TEMPLATE.read_text(encoding = "utf-8"))
+
+ assert result["success"] is True, result["errors"]
+ text = result["dataset"][0]["text"]
+ assert "response:get_weather{" in text
+ assert "response:get_time{" in text
+
+
+def test_sharegpt_repeated_call_with_only_boolean_arguments_is_not_lost():
+ first_call_only = (
+ "{%- for message in messages %}"
+ "{%- if message.tool_calls %}{%- set call = message.tool_calls[0].function %}"
+ "{{- '<call ' + call.name + '>' + call.arguments + '</call>' }}"
+ "{%- elif message.role == 'tool' %}{{- '<result from=set_light>' + message.content }}"
+ "{%- else %}{{- '<' + message.role + '>' + message.content }}{%- endif %}"
+ "{%- endfor %}"
+ )
+ call = json.dumps(
+ [
+ {"name": "set_light", "arguments": {"on": True}},
+ {"name": "set_light", "arguments": {"on": False}},
+ ]
+ )
+ row = _sharegpt_tool_row(call)
+ row["conversations"].insert(3, {"from": "observation", "value": '{"ok": 1}'})
+
+ result = _format_sharegpt([row], first_call_only)
+
+ text = result["dataset"][0]["text"]
+ assert '{"on": true}' in text and '{"on": false}' in text
+
+
+def test_sharegpt_call_is_kept_when_the_template_omits_the_function_name():
+ no_names = (
+ "{%- for message in messages %}"
+ "{%- if message.tool_calls %}{{- '<call>' + message.tool_calls[0].function.arguments }}"
+ "{%- else %}{{- '<' + message.role + '>' + message.content }}{%- endif %}"
+ "{%- endfor %}"
+ )
+ call = json.dumps({"name": "get_weather", "arguments": {"city": "Paris"}})
+
+ result = _format_sharegpt([_sharegpt_tool_row(call)], no_names)
+
+ assert "get_weather" in result["dataset"][0]["text"]
+
+
+def test_sharegpt_split_results_follow_their_own_call():
+ latest_call_names_results = (
+ "{%- set ns = namespace(last='') %}{%- for message in messages %}"
+ "{%- if message.tool_calls %}{%- set ns.last = message.tool_calls[0].function.name %}"
+ "{{- '<call ' + ns.last + '>' + message.tool_calls[0].function.arguments }}"
+ "{%- elif message.role == 'tool' %}{{- '<result from=' + ns.last + '>' + message.content }}"
+ "{%- else %}{{- '<' + message.role + '>' + message.content }}{%- endif %}"
+ "{%- endfor %}"
+ )
+ call = json.dumps(
+ [
+ {"name": "get_weather", "arguments": {"city": "Paris"}},
+ {"name": "get_time", "arguments": {"city": "Rome"}},
+ ]
+ )
+ row = _sharegpt_tool_row(call)
+ row["conversations"].insert(3, {"from": "observation", "value": '{"time": "noon"}'})
+
+ text = _format_sharegpt([row], latest_call_names_results)["dataset"][0]["text"]
+
+ assert (
+ text.index("<call get_weather>")
+ < text.index('{"temp": 18}')
+ < text.index("<call get_time>")
+ )
+ assert text.index("<call get_time>") < text.index('{"time": "noon"}')
+
+
+def test_sharegpt_split_calls_keep_null_content_for_templates_gating_on_it():
+ one_call_null_content = (
+ "{%- for message in messages %}"
+ "{%- if message.role == 'assistant' and message.content is none %}"
+ "{%- if message.tool_calls | length != 1 %}{{- raise_exception('one call') }}{%- endif %}"
+ "{{- '<call>' + message.tool_calls[0].function.name + message.tool_calls[0].function.arguments }}"
+ "{%- elif message.role == 'tool' %}{{- '<output>' + message.content }}"
+ "{%- elif message.role in ('user', 'assistant') %}{{- '<' + message.role + '>' + message.content }}"
+ "{%- endif %}{%- endfor %}"
+ )
+ call = json.dumps(
+ [
+ {"name": "get_weather", "arguments": {"city": "Paris"}},
+ {"name": "get_time", "arguments": {"city": "Rome"}},
+ ]
+ )
+ row = _sharegpt_tool_row(call)
+ row["conversations"].insert(3, {"from": "observation", "value": '{"time": "noon"}'})
+
+ text = _format_sharegpt([row], one_call_null_content)["dataset"][0]["text"]
+
+ assert '<call>get_weather{"city": "Paris"}' in text
+ assert '<call>get_time{"city": "Rome"}' in text
+
+
+def test_sharegpt_non_json_call_is_not_dropped_by_a_template_skipping_unknown_roles():
+ known_roles_only = (
+ "{%- for message in messages %}"
+ "{%- if message.role in ('user', 'assistant', 'tool') %}"
+ "{{- '<' + message.role + '>' + message.content }}{%- endif %}"
+ "{%- endfor %}"
+ )
+
+ result = _format_sharegpt([_sharegpt_tool_row("get_weather(Paris)")], known_roles_only)
+
+ assert "<tool>" not in result["dataset"][0]["text"]
diff --git a/studio/backend/utils/datasets/chat_templates.py b/studio/backend/utils/datasets/chat_templates.py
index f123d751623..b8a6cc3b525 100644
--- a/studio/backend/utils/datasets/chat_templates.py
+++ b/studio/backend/utils/datasets/chat_templates.py
@@ -200,7 +200,130 @@ def _row_tools(tools):
return normalized
+def _sharegpt_tool_turns(conversation, content = "", probe = False):
+ """Map ShareGPT ``function_call`` / ``observation`` turns to OpenAI tool turns, and the
+ markers ``probe`` put in place of each call's arguments and each result; the conversation
+ itself when it has neither role."""
+ if not any(
+ isinstance(message, dict) and message.get("role") in ("observation", "function_call")
+ for message in conversation
+ ):
+ return conversation, []
+ turns = []
+ markers = []
+ result_ids = []
+ calls_made = 0
+ for index, message in enumerate(conversation):
+ role = message.get("role") if isinstance(message, dict) else None
+ if role == "observation":
+ message = {**message, "role": "tool"}
+ if result_ids:
+ message["tool_call_id"] = result_ids.pop(0)
+ if probe:
+ markers.append(f"unslothresult{len(markers)}end")
+ message["content"] = markers[-1]
+ elif role == "function_call":
+ try:
+ calls = json.loads(message.get("content"))
+ except (TypeError, ValueError, RecursionError):
+ calls = None
+ calls = calls if isinstance(calls, list) else [calls]
+ if calls and all(isinstance(call, dict) and call.get("name") for call in calls):
+ tool_calls = []
+ for call in calls:
+ arguments = call.get("arguments", {})
+ # A JSON string keeps explicit nulls through _drop_none_values.
+ if not isinstance(arguments, str):
+ arguments = json.dumps(arguments, ensure_ascii = False)
+ name = call["name"]
+ if probe:
+ markers.append(f"unslothname{len(markers)}end")
+ name = markers[-1]
+ markers.append(f"unslothcall{len(markers)}end")
+ arguments = json.dumps({"probe": markers[-1]})
+ calls_made += 1
+ tool_calls.append(
+ {
+ # Nine alphanumerics, as Mistral requires.
+ "id": f"call{calls_made:05d}",
+ "type": "function",
+ "function": {"name": name, "arguments": arguments},
+ }
+ )
+ message = {"role": "assistant", "content": content, "tool_calls": tool_calls}
+ results = 0
+ for later in conversation[index + 1 :]:
+ if not (isinstance(later, dict) and later.get("role") == "observation"):
+ break
+ results += 1
+ # Results pair with calls by position only when there is one per call.
+ result_ids = [call["id"] for call in tool_calls] if results == len(calls) else []
+ else:
+ result_ids = []
+ if probe:
+ # Kept as written: a template skipping unknown roles must not pass on its result.
+ markers.append(f"unslothraw{len(markers)}end")
+ message = {**message, "content": markers[-1]}
+ turns.append(message)
+ return turns, markers
+
+
+def _one_call_per_message(turns):
+ # Each call then its own result: gpt-oss names a result after the latest call.
+ split = []
+ index = 0
+ while index < len(turns):
+ message = turns[index]
+ index += 1
+ calls = message.get("tool_calls") if isinstance(message, dict) else None
+ if not calls or len(calls) < 2:
+ split.append(message)
+ continue
+ run = []
+ while (
+ index < len(turns)
+ and isinstance(turns[index], dict)
+ and turns[index].get("role") == "tool"
+ ):
+ run.append(turns[index])
+ index += 1
+ results = {result.get("tool_call_id"): result for result in run}
+ paired = len(run) == len(calls) and all(call.get("id") in results for call in calls)
+ for i, call in enumerate(calls):
+ # Later pieces repeat no text, but keep a None content for templates gating on it.
+ content = message.get("content")
+ split.append({**message, "tool_calls": [call], "content": "" if i and content else content})
+ if paired:
+ split.append(results[call["id"]])
+ if not paired:
+ split.extend(run)
+ return split if len(split) != len(turns) else turns
+
+
def _render_conversation(tokenizer, conversation, tools = None, fallback_without_tools = True):
+ candidates = []
+ # None content for DeepSeek-style templates; one call per message for Llama 3.x and gpt-oss.
+ for content in ("", None):
+ turns, _ = _sharegpt_tool_turns(conversation, content)
+ if turns is conversation:
+ break
+ probe, markers = _sharegpt_tool_turns(conversation, content, probe = True)
+ candidates.append((turns, probe, markers))
+ split = _one_call_per_message(turns)
+ if split is not turns:
+ candidates.append((split, _one_call_per_message(probe), markers))
+ for turns, probe, markers in candidates:
+ # Templates may ignore tool_calls, drop tool turns, or render only the first call (gpt-oss).
+ try:
+ shown = _render_messages(tokenizer, probe, tools, fallback_without_tools)
+ if all(marker in shown for marker in markers):
+ return _render_messages(tokenizer, turns, tools, fallback_without_tools)
+ except Exception:
+ pass
+ return _render_messages(tokenizer, conversation, tools, fallback_without_tools)
+
+
+def _render_messages(tokenizer, conversation, tools = None, fallback_without_tools = True):
from core.inference.chat_template_helpers import _normalize_tool_call_arguments
attempts = []
@@ -220,7 +343,7 @@ def _render_conversation(tokenizer, conversation, tools = None, fallback_without
if first_error is None:
first_error = error
if tools and fallback_without_tools:
- return _render_conversation(tokenizer, conversation)
+ return _render_messages(tokenizer, conversation)
raise first_error