Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""#1217: `format: tool-calling` reads the calls where agent data puts them.
`soup agent synth` / `soup agent train` write each call inside its assistant
turn (the OpenAI shape), but the tool-calling converter only read top-level
`tools` / `tool_calls` keys, so every synthesised row was dropped and
`soup train --dry-run` still printed "Ready to train!". In rows it did accept,
the call was re-appended after the final answer and `tool_call_id` was lost.
The converter now keeps the calls on their own turns (the top-level list is
the legacy single-call form), `tools` is optional, `detect_format` classifies
`{messages, tools}` as tool-calling, and synth rows carry a `tools` schema.
The zero-row stop is covered in test_issue1217_zero_row_stop.py.
"""
from __future__ import annotations
import json
import re
from pathlib import Path
from types import SimpleNamespace
import pytest
from typer.testing import CliRunner
from soup_cli.cli import app
from soup_cli.config.schema import DataConfig
from soup_cli.data.formats import (
detect_format,
format_to_messages,
format_to_messages_with_reason,
)
from soup_cli.data.loader import load_dataset
from soup_cli.utils.agent_forge import (
Endpoint,
endpoint_to_rows,
synthesise_dataset,
write_dataset,
)
from tests.conftest import strip_ansi
def _local_base_model_dir() -> str:
"""A tiny Llama causal LM + tokenizer built locally (no download),
standing in for hf-internal-testing/tiny-random-LlamaForCausalLM -- #1356
found this pulling it from the Hub for a dry-run plan and a tokenizer
whose chat_template gets overwritten immediately after load. Shared and
cached across every file that needs one (tests/_tiny_hf_models.py)."""
from tests._tiny_hf_models import tiny_model_dir
return tiny_model_dir("llama")
_WEATHER_TOOL = {
"type": "function",
"function": {
"name": "get_weather",
"description": "Weather for a city",
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}},
},
}
_SCHEMA_TURN = {
"role": "system",
"content": (
"You have access to the following tools. When a tool call is needed, "
"respond with a function call in JSON.\n\n"
"- get_weather: Weather for a city\n"
' parameters: {"type": "object", "properties": {"city": {"type": "string"}}}'
),
}
def _call(call_id: str, city: str, *, as_dict: bool = False) -> dict:
arguments = {"city": city} if as_dict else json.dumps({"city": city})
return {
"id": call_id,
"type": "function",
"function": {"name": "get_weather", "arguments": arguments},
}
def _normalized_call(call_id: str, city: str) -> dict:
return {
"id": call_id,
"type": "function",
"function": {"name": "get_weather", "arguments": json.dumps({"city": city})},
}
def _plain(text: str) -> str:
return re.sub(r"\s+", " ", strip_ansi(text)).strip()
def _write_jsonl(path: Path, rows: list[dict]) -> None:
path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8")
def _loaded_rows(path: str, data_format: str) -> list[dict]:
"""The rows `soup train` would train on."""
config = DataConfig(train=path, format=data_format, val_split=0.0)
return list(load_dataset(config)["train"])
# --- The synthesised dataset loads with the recipe `soup agent train` prints ---
_ENDPOINTS = [
Endpoint(tool="listPets", method="get", path="/pets", description="List pets",
parameters=(), spec_kind="openapi"),
Endpoint(tool="getPet", method="get", path="/pets/{id}", description="Get a pet",
parameters=("id", "fields"), spec_kind="openapi"),
]
@pytest.mark.parametrize("data_format", ["tool-calling", "auto"])
def test_every_synthesised_row_loads_with_its_tool_call(
tmp_path, monkeypatch, data_format
):
monkeypatch.chdir(tmp_path)
rows = synthesise_dataset(_ENDPOINTS, examples_per_endpoint=3)
write_dataset(rows, "agent_dataset.jsonl")
loaded = _loaded_rows("agent_dataset.jsonl", data_format)
assert len(loaded) == len(rows) == 6
for source, got in zip(rows, loaded):
calls = [m for m in got["messages"] if m.get("tool_calls")]
assert len(calls) == 1
assert calls[0]["tool_calls"][0]["function"]["name"] == source.tool
assert got["messages"][0]["role"] == "system"
assert f"- {source.tool}: " in got["messages"][0]["content"]
def test_the_printed_recipe_dry_runs_with_the_synthesised_rows(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
Path("openapi.yaml").write_text(
"openapi: 3.0.0\ninfo: {title: pets, version: '1'}\npaths:\n"
" /pets:\n get:\n operationId: listPets\n summary: List pets\n",
encoding="utf-8",
)
runner = CliRunner()
base = Path(_local_base_model_dir()).as_posix()
planned = runner.invoke(app, ["agent", "train", "--spec", "openapi.yaml", "--base", base])
assert planned.exit_code == 0, planned.output
assert "format: tool-calling" in planned.output
Path("agent_train.yaml").write_text(
f"base: {base}\ntask: sft\ndata:\n train: agent_dataset.jsonl\n"
" format: tool-calling\n val_split: 0.0\n"
"training:\n epochs: 3\n lr: 2.0e-5\n batch_size: auto\n"
"output: ./agent_train_output\n",
encoding="utf-8",
)
result = runner.invoke(app, ["train", "--config", "agent_train.yaml", "--dry-run"])
output = _plain(result.output)
assert result.exit_code == 0, output
assert "Data OK: 4 train samples" in output
assert "dropped" not in output
def test_a_synth_row_carries_the_schema_of_its_endpoint():
row = endpoint_to_rows(_ENDPOINTS[1], 1)[0].to_dict()
assert row["tools"] == [{
"type": "function",
"function": {
"name": "getPet",
"description": "Get a pet",
"parameters": {"type": "object", "properties": {"id": {}, "fields": {}}},
},
}]
assert detect_format([row]) == "tool-calling"
# --- Each row shape, converted exactly --------------------------------------
def test_call_nested_in_the_assistant_turn_without_top_level_keys():
row = {"messages": [
{"role": "user", "content": "Weather in Paris?"},
{"role": "assistant", "tool_calls": [_call("call_0", "Paris")]},
]}
assert format_to_messages(row, "tool-calling") == {"messages": [
{"role": "user", "content": "Weather in Paris?"},
{"role": "assistant", "content": "", "tool_calls": [_normalized_call("call_0", "Paris")]},
]}
def test_the_openai_fine_tuning_shape():
row = {
"messages": [
{"role": "user", "content": "Weather in Paris?"},
{"role": "assistant", "content": None, "tool_calls": [_call("call_0", "Paris")]},
{"role": "tool", "tool_call_id": "call_0", "content": '{"temp": 21}'},
{"role": "assistant", "content": "It is 21 C in Paris."},
],
"tools": [_WEATHER_TOOL],
}
assert format_to_messages(row, "tool-calling") == {"messages": [
_SCHEMA_TURN,
{"role": "user", "content": "Weather in Paris?"},
{"role": "assistant", "content": "", "tool_calls": [_normalized_call("call_0", "Paris")]},
{"role": "tool", "content": '{"temp": 21}', "tool_call_id": "call_0"},
{"role": "assistant", "content": "It is 21 C in Paris."},
]}
def test_the_documented_top_level_shape_still_converts():
row = {
"messages": [{"role": "user", "content": "Weather in Paris?"}],
"tools": [_WEATHER_TOOL],
"tool_calls": [{"function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}}],
}
assert format_to_messages(row, "tool-calling") == {"messages": [
_SCHEMA_TURN,
{"role": "user", "content": "Weather in Paris?"},
{"role": "assistant", "content": "", "tool_calls": [
{"function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}},
]},
]}
def test_two_tool_rounds_keep_source_order_and_their_ids():
messages = [
{"role": "user", "content": "Paris, then Rome?"},
{"role": "assistant", "content": "", "tool_calls": [_call("call_0", "Paris")]},
{"role": "tool", "tool_call_id": "call_0", "content": "21"},
{"role": "assistant", "content": "", "tool_calls": [_call("call_1", "Rome")]},
{"role": "tool", "tool_call_id": "call_1", "content": "25"},
{"role": "assistant", "content": "Paris 21 C, Rome 25 C."},
]
# The top-level list repeats the calls, as the issue's documented-shape
# row does; it must not be appended after the final answer again.
row = {
"messages": messages,
"tools": [_WEATHER_TOOL],
"tool_calls": [_call("call_0", "Paris")],
}
converted = format_to_messages(row, "tool-calling")["messages"]
assert [m["role"] for m in converted] == [
"system", "user", "assistant", "tool", "assistant", "tool", "assistant",
]
assert converted[2]["tool_calls"] == [_normalized_call("call_0", "Paris")]
assert converted[3]["tool_call_id"] == "call_0"
assert converted[4]["tool_calls"] == [_normalized_call("call_1", "Rome")]
assert converted[5]["tool_call_id"] == "call_1"
assert converted[6] == {"role": "assistant", "content": "Paris 21 C, Rome 25 C."}
def test_two_parallel_calls_in_one_assistant_turn():
row = {
"messages": [
{"role": "user", "content": "Paris and Rome?"},
{"role": "assistant", "content": "",
"tool_calls": [_call("call_0", "Paris"), _call("call_1", "Rome")]},
{"role": "tool", "tool_call_id": "call_0", "content": "21"},
{"role": "tool", "tool_call_id": "call_1", "content": "25"},
{"role": "assistant", "content": "21 and 25."},
],
"tools": [_WEATHER_TOOL],
}
converted = format_to_messages(row, "tool-calling")["messages"]
assert converted[2]["tool_calls"] == [
_normalized_call("call_0", "Paris"), _normalized_call("call_1", "Rome"),
]
assert [m.get("tool_call_id") for m in converted[3:5]] == ["call_0", "call_1"]
assert converted[-1] == {"role": "assistant", "content": "21 and 25."}
@pytest.mark.parametrize("as_dict", [False, True], ids=["json-string", "dict"])
def test_arguments_as_a_json_string_or_a_dict(as_dict):
row = {"messages": [
{"role": "user", "content": "Weather in Paris?"},
{"role": "assistant", "content": "", "tool_calls": [_call("c", "Paris", as_dict=as_dict)]},
]}
converted = format_to_messages(row, "tool-calling")["messages"]
assert converted[1]["tool_calls"][0]["function"]["arguments"] == '{"city": "Paris"}'
@pytest.mark.parametrize("tools", [None, []], ids=["absent", "empty"])
def test_without_tools_there_is_no_schema_turn(tools):
row = {"messages": [
{"role": "system", "content": "Be brief."},
{"role": "user", "content": "Weather in Paris?"},
{"role": "assistant", "content": "", "tool_calls": [_call("c", "Paris")]},
]}
if tools is not None:
row["tools"] = tools
converted = format_to_messages(row, "tool-calling")["messages"]
assert converted[0] == {"role": "system", "content": "Be brief."}
assert [m["role"] for m in converted] == ["system", "user", "assistant"]
def test_with_tools_the_rows_system_prompt_is_merged_into_the_schema_turn():
row = {
"messages": [
{"role": "system", "content": "Be brief."},
{"role": "user", "content": "Weather in Paris?"},
],
"tools": [_WEATHER_TOOL],
}
converted = format_to_messages(row, "tool-calling")["messages"]
assert converted[0] == {
"role": "system", "content": "Be brief.\n\n" + _SCHEMA_TURN["content"],
}
assert [m["role"] for m in converted] == ["system", "user"]
_OPENAI_ROW = {
"messages": [
{"role": "user", "content": "Weather in Paris?"},
{"role": "assistant", "content": "", "tool_calls": [_call("call_0", "Paris")]},
{"role": "tool", "tool_call_id": "call_0", "content": "21"},
{"role": "assistant", "content": "21 C."},
],
"tools": [_WEATHER_TOOL],
}
def test_auto_detects_the_openai_shape_and_converts_it_like_the_explicit_format(tmp_path):
path = tmp_path / "agent.jsonl"
_write_jsonl(path, [_OPENAI_ROW])
assert detect_format([_OPENAI_ROW]) == "tool-calling"
assert _loaded_rows(str(path), "auto") == _loaded_rows(str(path), "tool-calling")
def test_auto_keeps_nested_calls_when_the_row_has_no_tools(tmp_path):
# No top-level "tools": the row is plain chatml, which keeps the messages
# (and their calls) as written.
row = {"messages": _OPENAI_ROW["messages"]}
path = tmp_path / "chat.jsonl"
_write_jsonl(path, [row])
assert _loaded_rows(str(path), "auto") == [row]
def test_tool_calling_is_still_detected_before_audio():
row = {**_OPENAI_ROW, "audio": "a.wav"}
assert detect_format([row]) == "tool-calling"
def test_tools_as_a_json_string_convert_like_a_list(tmp_path):
row = {**_OPENAI_ROW, "tools": json.dumps([_WEATHER_TOOL])}
path = tmp_path / "agent.jsonl"
_write_jsonl(path, [row])
assert format_to_messages(row, "tool-calling") == format_to_messages(
_OPENAI_ROW, "tool-calling"
)
assert len(_loaded_rows(str(path), "auto")) == 1
def test_tools_as_a_string_that_is_not_json_drops_the_row():
row = {**_OPENAI_ROW, "tools": "{not json"}
converted, why = format_to_messages_with_reason(row, "tool-calling")
assert converted is None
assert "'tools' is a string that is not JSON" in why
# --- Null content is only allowed on a call-only assistant turn --------------
# A call-only turn with null content converting to "" is pinned by
# test_the_openai_fine_tuning_shape above.
@pytest.mark.parametrize(
"turn",
[
{"role": "assistant", "content": None, "tool_calls": []},
{"role": "assistant", "content": None},
{"role": "user", "content": None},
{"role": "tool", "tool_call_id": "call_0", "content": None},
],
ids=["assistant-empty-calls", "assistant-no-calls", "user", "tool"],
)
def test_null_content_outside_a_call_turn_drops_the_row(turn):
# A template printing {{ message.content }} would render the literal "None".
row = {"messages": [{"role": "user", "content": "q"}, turn]}
converted, why = format_to_messages_with_reason(row, "tool-calling")
assert converted is None
assert f"tool-calling {turn['role']} content must be a string" in why
# --- Malformed nested calls are still dropped, with a reason -----------------
@pytest.mark.parametrize(
("assistant", "reason"),
[
({"role": "assistant", "tool_calls": [
{"function": {"name": "get_weather", "arguments": "{bad"}}]},
"must be JSON-parseable"),
({"role": "assistant", "tool_calls": {"function": {"name": "f"}}},
"'messages.tool_calls' must be a list"),
({"role": "assistant", "tool_calls": [{"function": {"name": ""}}]},
"'function.name' must be a non-empty string"),
({"role": "assistant", "tool_calls": [
{"id": 7, "function": {"name": "f", "arguments": "{}"}}]},
"'id' must be a string"),
],
ids=["bad-json", "not-a-list", "empty-name", "non-string-id"],
)
def test_a_malformed_nested_call_drops_the_row(assistant, reason):
row = {"messages": [{"role": "user", "content": "q"}, assistant]}
converted, why = format_to_messages_with_reason(row, "tool-calling")
assert converted is None
assert reason in why
def test_a_non_string_tool_call_id_drops_the_row():
row = {"messages": [
{"role": "user", "content": "q"},
{"role": "tool", "tool_call_id": 3, "content": "21"},
]}
converted, why = format_to_messages_with_reason(row, "tool-calling")
assert converted is None
assert "'tool_call_id' must be a string" in why
# --- The rendered training text, through Arrow and a tool-aware template -------
# The Llama 3.x templates branch on `'tool_calls' in message`, so they see a
# key that is present with a None value as a call.
_LLAMA_STYLE_TEMPLATE = (
"{%- for message in messages %}"
"{%- if message.role == 'tool' %}"
"<tool id={{ message.tool_call_id }}>{{ message.content }}</tool>"
"{%- elif 'tool_calls' in message %}"
"{%- if message.tool_calls|length != 1 %}"
"{{ raise_exception('one call per turn') }}"
"{%- endif %}"
"{% generation %}<call>{{ message.tool_calls[0].function.name }}"
"({{ message.tool_calls[0].function.arguments }})</call>{% endgeneration %}"
"{%- elif message.role == 'assistant' %}"
"{% generation %}<a>{{ message.content }}</a>{% endgeneration %}"
"{%- else %}<{{ message.role }}>{{ message.content }}</{{ message.role }}>"
"{%- endif %}"
"{%- endfor %}"
)
def test_the_trajectory_renders_in_source_order_through_arrow():
from datasets import Dataset
from transformers import AutoTokenizer
from soup_cli.data.sft_format import build_format_row
tokenizer = AutoTokenizer.from_pretrained(_local_base_model_dir())
tokenizer.chat_template = _LLAMA_STYLE_TEMPLATE
data_cfg = SimpleNamespace(
chat_template=None,
train_on_responses_only=True,
train_on_messages_with_train_field=False,
max_length=512,
)
row = format_to_messages({"messages": _OPENAI_ROW["messages"]}, "tool-calling")
# The trainer maps rows through Arrow, which gives every message the
# sparse keys it lacks as None (a user turn gets "tool_calls": None).
arrow_rows = Dataset.from_list([row])
assert arrow_rows[0]["messages"][0]["tool_calls"] is None
formatted = arrow_rows.map(
build_format_row(tokenizer, data_cfg), remove_columns=["messages"]
)
text = tokenizer.decode(formatted[0]["input_ids"], skip_special_tokens=True)
assert re.sub(r"\s+", "", text) == re.sub(r"\s+", "", (
'<user>Weather in Paris?</user><call>get_weather({"city": "Paris"})</call>'
"<tool id=call_0>21</tool><a>21 C.</a>"
))