Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: end Alpaca training samples with the end token (#11719)

Nilaycbf92d12f96a
diff --git a/studio/backend/tests/test_alpaca_blank_cells.py b/studio/backend/tests/test_alpaca_blank_cells.py
index 78649c3c7c6..a30f1087fb2 100644
--- a/studio/backend/tests/test_alpaca_blank_cells.py
+++ b/studio/backend/tests/test_alpaca_blank_cells.py
@@ -12,17 +12,21 @@ class _Tokenizer:
eos_token = "</s>"
-def _format(dataset):
+def _format_result(dataset, tokenizer = None):
result = format_and_template_dataset(
dataset,
model_name = "Qwen2ForCausalLM",
- tokenizer = _Tokenizer(),
+ tokenizer = tokenizer or _Tokenizer(),
batch_size = 2,
num_proc = 1,
)
assert result["success"] is True
assert result["final_format"] == "alpaca"
- return list(result["dataset"]["text"])
+ return result
+
+
+def _format(dataset, tokenizer = None):
+ return list(_format_result(dataset, tokenizer)["dataset"]["text"])
def test_missing_alpaca_values_are_formatted_as_empty_text():
@@ -36,8 +40,8 @@ def test_missing_alpaca_values_are_formatted_as_empty_text():
texts = _format(dataset)
- assert texts[0].endswith("### Instruction:\nSay hi\n\n### Input:\n\n\n### Response:\nhi")
- assert texts[1].endswith("### Instruction:\n\n\n### Input:\ncontext\n\n### Response:\n")
+ assert texts[0].endswith("### Instruction:\nSay hi\n\n### Input:\n\n\n### Response:\nhi</s>")
+ assert texts[1].endswith("### Instruction:\n\n\n### Input:\ncontext\n\n### Response:\n</s>")
assert all("None" not in text for text in texts)
@@ -48,11 +52,38 @@ def test_blank_csv_cells_are_not_trained_as_none(tmp_path):
texts = _format(dataset)
- assert texts[0].endswith("### Input:\n\n\n### Response:\nhi")
- assert texts[1].endswith("### Input:\n\n\n### Response:\n")
+ assert texts[0].endswith("### Input:\n\n\n### Response:\nhi</s>")
+ assert texts[1].endswith("### Input:\n\n\n### Response:\n</s>")
assert all("None" not in text for text in texts)
+def test_alpaca_rows_do_not_get_a_second_eos():
+ dataset = Dataset.from_dict({"instruction": ["Echo"], "input": [""], "output": ["done</s>"]})
+
+ assert _format(dataset)[0].endswith("### Response:\ndone</s>")
+
+
+def test_alpaca_rows_take_eos_from_a_processors_tokenizer():
+ class _Processor:
+ chat_template = "{{ messages }}"
+ tokenizer = _Tokenizer()
+
+ dataset = Dataset.from_dict({"instruction": ["Add 2+2"], "input": [""], "output": ["4"]})
+
+ assert _format(dataset, _Processor())[0].endswith("### Response:\n4</s>")
+
+
+def test_alpaca_warns_when_tokenizer_has_no_eos():
+ class _NoEosTokenizer:
+ chat_template = "{{ messages }}"
+
+ dataset = Dataset.from_dict({"instruction": ["Add 2+2"], "input": [""], "output": ["4"]})
+ result = _format_result(dataset, _NoEosTokenizer())
+
+ assert result["dataset"][0]["text"].endswith("### Response:\n4")
+ assert any("no eos_token" in warning for warning in result["warnings"])
+
+
def test_blank_csv_cells_are_not_converted_to_none_for_chatml(tmp_path):
csv_path = tmp_path / "train.csv"
csv_path.write_text("instruction,input,output\nSay bye,,\n,context,answer\n")
diff --git a/studio/backend/tests/test_chatml_to_alpaca.py b/studio/backend/tests/test_chatml_to_alpaca.py
index 67a05388a53..618ea950afd 100644
--- a/studio/backend/tests/test_chatml_to_alpaca.py
+++ b/studio/backend/tests/test_chatml_to_alpaca.py
@@ -186,5 +186,5 @@ def test_alpaca_format_trains_on_every_sharegpt_exchange():
assert len(texts) == 6
assert texts[5].endswith(
"### Instruction:\nQ3-2\n\n### Input:\nSYS-2\n\nUser: Q1-2\nAssistant: A1-2\n"
- "User: Q2-2\nAssistant: A2-2\n\n### Response:\nA3-2"
+ "User: Q2-2\nAssistant: A2-2\n\n### Response:\nA3-2</s>"
)
diff --git a/studio/backend/tests/test_dataset_custom_prompt_template.py b/studio/backend/tests/test_dataset_custom_prompt_template.py
index ff992a39ec1..3e0dc6b695b 100644
--- a/studio/backend/tests/test_dataset_custom_prompt_template.py
+++ b/studio/backend/tests/test_dataset_custom_prompt_template.py
@@ -81,7 +81,7 @@ def test_default_template_is_unchanged_when_parameter_is_omitted():
assert result["dataset"][0]["text"] == (
"Below is an instruction that describes a task, paired with an input that provides "
"further context. Write a response that appropriately completes the request.\n\n"
- "### Instruction:\nWhat is 2+2?\n\n### Input:\n\n\n### Response:\n4"
+ "### Instruction:\nWhat is 2+2?\n\n### Input:\n\n\n### Response:\n4</s>"
)
@@ -130,7 +130,7 @@ def test_main_entry_point_keeps_default_alpaca_output():
assert result["final_format"] == "alpaca"
assert len(result["dataset"]) == 1
assert result["dataset"].column_names == dataset.column_names + ["text"]
- assert result["dataset"][0]["text"].endswith("### Response:\n4")
+ assert result["dataset"][0]["text"].endswith("### Response:\n4</s>")
def test_main_entry_point_keeps_raw_text_path_unchanged():
diff --git a/studio/backend/utils/datasets/chat_templates.py b/studio/backend/utils/datasets/chat_templates.py
index 60b1d72c49c..e80256dfb28 100644
--- a/studio/backend/utils/datasets/chat_templates.py
+++ b/studio/backend/utils/datasets/chat_templates.py
@@ -148,7 +148,7 @@ def apply_chat_template_to_dataset(
):
"""Apply the chat template to a dataset based on its format, returning a dict with the dataset, success status, warnings and errors.
- ``dataset_info`` is the output of format_dataset() with metadata. ``custom_prompt_template`` is deprecated and non-None values are rejected, because Studio cannot persist a matching inference template. ``add_eos_token`` appends tokenizer.eos_token to each text, ``remove_bos_prefix`` strips a leading '<bos>' (Gemma and friends), ``custom_format_mapping`` maps custom columns to the standard format, and ``batch_size`` / ``num_proc`` control processing.
+ ``dataset_info`` is the output of format_dataset() with metadata. ``custom_prompt_template`` is deprecated and non-None values are rejected, because Studio cannot persist a matching inference template. ``add_eos_token`` appends the tokenizer's eos_token to each ChatML text (Alpaca text always gets one), ``remove_bos_prefix`` strips a leading '<bos>' (Gemma and friends), ``custom_format_mapping`` maps custom columns to the standard format, and ``batch_size`` / ``num_proc`` control processing.
"""
dataset = dataset_info["dataset"]
final_format = dataset_info["final_format"]
@@ -168,12 +168,14 @@ def apply_chat_template_to_dataset(
"errors": errors,
}
- eos_token = ""
- if add_eos_token:
- if hasattr(tokenizer, 'eos_token') and tokenizer.eos_token:
- eos_token = tokenizer.eos_token
- else:
- warnings.append("add_eos_token=True but tokenizer has no eos_token")
+ # A processor (Gemma 3 on the text path) keeps eos_token on its inner tokenizer.
+ eos_token = (
+ getattr(tokenizer, 'eos_token', None)
+ or getattr(getattr(tokenizer, 'tokenizer', None), 'eos_token', None)
+ or ""
+ )
+ if not eos_token and (add_eos_token or final_format == "alpaca"):
+ warnings.append("Tokenizer has no eos_token, so EOS was not appended")
# CUSTOM FORMAT MAPPING (for non-standard datasets)
if final_format == "unknown":
@@ -292,7 +294,9 @@ def apply_chat_template_to_dataset(
text = DEFAULT_ALPACA_TEMPLATE.format(
fields["instruction"], fields["input"], fields["output"]
)
- texts.append(text + eos_token)
+ if not text.endswith(eos_token):
+ text += eos_token
+ texts.append(text)
return {"text": texts}
@@ -367,7 +371,8 @@ def apply_chat_template_to_dataset(
if remove_bos_prefix:
text = text.removeprefix('<bos>')
- text += eos_token
+ if add_eos_token:
+ text += eos_token
texts.append(text)
row_errors.append("")