Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: end Alpaca training samples with the end token (#11719)
Nilaycbf92d12f96a
diff --git a/studio/backend/tests/test_alpaca_blank_cells.py b/studio/backend/tests/test_alpaca_blank_cells.pyindex 78649c3c7c6..a30f1087fb2 100644--- a/studio/backend/tests/test_alpaca_blank_cells.py+++ b/studio/backend/tests/test_alpaca_blank_cells.py@@ -12,17 +12,21 @@ class _Tokenizer:eos_token = "</s>"-def _format(dataset):+def _format_result(dataset, tokenizer = None):result = format_and_template_dataset(dataset,model_name = "Qwen2ForCausalLM",- tokenizer = _Tokenizer(),+ tokenizer = tokenizer or _Tokenizer(),batch_size = 2,num_proc = 1,)assert result["success"] is Trueassert result["final_format"] == "alpaca"- return list(result["dataset"]["text"])+ return result+++def _format(dataset, tokenizer = None):+ return list(_format_result(dataset, tokenizer)["dataset"]["text"])def test_missing_alpaca_values_are_formatted_as_empty_text():@@ -36,8 +40,8 @@ def test_missing_alpaca_values_are_formatted_as_empty_text():texts = _format(dataset)- assert texts[0].endswith("### Instruction:\nSay hi\n\n### Input:\n\n\n### Response:\nhi")- assert texts[1].endswith("### Instruction:\n\n\n### Input:\ncontext\n\n### Response:\n")+ assert texts[0].endswith("### Instruction:\nSay hi\n\n### Input:\n\n\n### Response:\nhi</s>")+ assert texts[1].endswith("### Instruction:\n\n\n### Input:\ncontext\n\n### Response:\n</s>")assert all("None" not in text for text in texts)@@ -48,11 +52,38 @@ def test_blank_csv_cells_are_not_trained_as_none(tmp_path):texts = _format(dataset)- assert texts[0].endswith("### Input:\n\n\n### Response:\nhi")- assert texts[1].endswith("### Input:\n\n\n### Response:\n")+ assert texts[0].endswith("### Input:\n\n\n### Response:\nhi</s>")+ assert texts[1].endswith("### Input:\n\n\n### Response:\n</s>")assert all("None" not in text for text in texts)+def test_alpaca_rows_do_not_get_a_second_eos():+ dataset = Dataset.from_dict({"instruction": ["Echo"], "input": [""], "output": ["done</s>"]})++ assert _format(dataset)[0].endswith("### Response:\ndone</s>")+++def test_alpaca_rows_take_eos_from_a_processors_tokenizer():+ class _Processor:+ chat_template = "{{ messages }}"+ tokenizer = _Tokenizer()++ dataset = Dataset.from_dict({"instruction": ["Add 2+2"], "input": [""], "output": ["4"]})++ assert _format(dataset, _Processor())[0].endswith("### Response:\n4</s>")+++def test_alpaca_warns_when_tokenizer_has_no_eos():+ class _NoEosTokenizer:+ chat_template = "{{ messages }}"++ dataset = Dataset.from_dict({"instruction": ["Add 2+2"], "input": [""], "output": ["4"]})+ result = _format_result(dataset, _NoEosTokenizer())++ assert result["dataset"][0]["text"].endswith("### Response:\n4")+ assert any("no eos_token" in warning for warning in result["warnings"])++def test_blank_csv_cells_are_not_converted_to_none_for_chatml(tmp_path):csv_path = tmp_path / "train.csv"csv_path.write_text("instruction,input,output\nSay bye,,\n,context,answer\n")diff --git a/studio/backend/tests/test_chatml_to_alpaca.py b/studio/backend/tests/test_chatml_to_alpaca.pyindex 67a05388a53..618ea950afd 100644--- a/studio/backend/tests/test_chatml_to_alpaca.py+++ b/studio/backend/tests/test_chatml_to_alpaca.py@@ -186,5 +186,5 @@ def test_alpaca_format_trains_on_every_sharegpt_exchange():assert len(texts) == 6assert texts[5].endswith("### Instruction:\nQ3-2\n\n### Input:\nSYS-2\n\nUser: Q1-2\nAssistant: A1-2\n"- "User: Q2-2\nAssistant: A2-2\n\n### Response:\nA3-2"+ "User: Q2-2\nAssistant: A2-2\n\n### Response:\nA3-2</s>")diff --git a/studio/backend/tests/test_dataset_custom_prompt_template.py b/studio/backend/tests/test_dataset_custom_prompt_template.pyindex ff992a39ec1..3e0dc6b695b 100644--- a/studio/backend/tests/test_dataset_custom_prompt_template.py+++ b/studio/backend/tests/test_dataset_custom_prompt_template.py@@ -81,7 +81,7 @@ def test_default_template_is_unchanged_when_parameter_is_omitted():assert result["dataset"][0]["text"] == ("Below is an instruction that describes a task, paired with an input that provides ""further context. Write a response that appropriately completes the request.\n\n"- "### Instruction:\nWhat is 2+2?\n\n### Input:\n\n\n### Response:\n4"+ "### Instruction:\nWhat is 2+2?\n\n### Input:\n\n\n### Response:\n4</s>")@@ -130,7 +130,7 @@ def test_main_entry_point_keeps_default_alpaca_output():assert result["final_format"] == "alpaca"assert len(result["dataset"]) == 1assert result["dataset"].column_names == dataset.column_names + ["text"]- assert result["dataset"][0]["text"].endswith("### Response:\n4")+ assert result["dataset"][0]["text"].endswith("### Response:\n4</s>")def test_main_entry_point_keeps_raw_text_path_unchanged():diff --git a/studio/backend/utils/datasets/chat_templates.py b/studio/backend/utils/datasets/chat_templates.pyindex 60b1d72c49c..e80256dfb28 100644--- a/studio/backend/utils/datasets/chat_templates.py+++ b/studio/backend/utils/datasets/chat_templates.py@@ -148,7 +148,7 @@ def apply_chat_template_to_dataset():"""Apply the chat template to a dataset based on its format, returning a dict with the dataset, success status, warnings and errors.- ``dataset_info`` is the output of format_dataset() with metadata. ``custom_prompt_template`` is deprecated and non-None values are rejected, because Studio cannot persist a matching inference template. ``add_eos_token`` appends tokenizer.eos_token to each text, ``remove_bos_prefix`` strips a leading '<bos>' (Gemma and friends), ``custom_format_mapping`` maps custom columns to the standard format, and ``batch_size`` / ``num_proc`` control processing.+ ``dataset_info`` is the output of format_dataset() with metadata. ``custom_prompt_template`` is deprecated and non-None values are rejected, because Studio cannot persist a matching inference template. ``add_eos_token`` appends the tokenizer's eos_token to each ChatML text (Alpaca text always gets one), ``remove_bos_prefix`` strips a leading '<bos>' (Gemma and friends), ``custom_format_mapping`` maps custom columns to the standard format, and ``batch_size`` / ``num_proc`` control processing."""dataset = dataset_info["dataset"]final_format = dataset_info["final_format"]@@ -168,12 +168,14 @@ def apply_chat_template_to_dataset("errors": errors,}- eos_token = ""- if add_eos_token:- if hasattr(tokenizer, 'eos_token') and tokenizer.eos_token:- eos_token = tokenizer.eos_token- else:- warnings.append("add_eos_token=True but tokenizer has no eos_token")+ # A processor (Gemma 3 on the text path) keeps eos_token on its inner tokenizer.+ eos_token = (+ getattr(tokenizer, 'eos_token', None)+ or getattr(getattr(tokenizer, 'tokenizer', None), 'eos_token', None)+ or ""+ )+ if not eos_token and (add_eos_token or final_format == "alpaca"):+ warnings.append("Tokenizer has no eos_token, so EOS was not appended")# CUSTOM FORMAT MAPPING (for non-standard datasets)if final_format == "unknown":@@ -292,7 +294,9 @@ def apply_chat_template_to_dataset(text = DEFAULT_ALPACA_TEMPLATE.format(fields["instruction"], fields["input"], fields["output"])- texts.append(text + eos_token)+ if not text.endswith(eos_token):+ text += eos_token+ texts.append(text)return {"text": texts}@@ -367,7 +371,8 @@ def apply_chat_template_to_dataset(if remove_bos_prefix:text = text.removeprefix('<bos>')- text += eos_token+ if add_eos_token:+ text += eos_tokentexts.append(text)row_errors.append("")