Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: Stop training the word None into CSV rows with blank cells (#10813)
Nilay9cfaa9dcdeee
diff --git a/studio/backend/tests/test_alpaca_blank_cells.py b/studio/backend/tests/test_alpaca_blank_cells.pynew file mode 100644index 00000000000..78649c3c7c6--- /dev/null+++ b/studio/backend/tests/test_alpaca_blank_cells.py@@ -0,0 +1,122 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++from datasets import Dataset, load_dataset++from utils.datasets import convert_alpaca_to_chatml, format_and_template_dataset+from utils.datasets.cells import cell_text+++class _Tokenizer:+ chat_template = "{{ messages }}"+ eos_token = "</s>"+++def _format(dataset):+ result = format_and_template_dataset(+ dataset,+ model_name = "Qwen2ForCausalLM",+ tokenizer = _Tokenizer(),+ batch_size = 2,+ num_proc = 1,+ )+ assert result["success"] is True+ assert result["final_format"] == "alpaca"+ return list(result["dataset"]["text"])+++def test_missing_alpaca_values_are_formatted_as_empty_text():+ dataset = Dataset.from_dict(+ {+ "instruction": ["Say hi", None],+ "input": [None, "context"],+ "output": ["hi", None],+ }+ )++ texts = _format(dataset)++ assert texts[0].endswith("### Instruction:\nSay hi\n\n### Input:\n\n\n### Response:\nhi")+ assert texts[1].endswith("### Instruction:\n\n\n### Input:\ncontext\n\n### Response:\n")+ assert all("None" not in text for text in texts)+++def test_blank_csv_cells_are_not_trained_as_none(tmp_path):+ csv_path = tmp_path / "train.csv"+ csv_path.write_text('instruction,input,output\nSay hi,,hi\nSay bye,"",\n')+ dataset = load_dataset("csv", data_files = str(csv_path), split = "train")++ texts = _format(dataset)++ assert texts[0].endswith("### Input:\n\n\n### Response:\nhi")+ assert texts[1].endswith("### Input:\n\n\n### Response:\n")+ assert all("None" not in text for text in texts)+++def test_blank_csv_cells_are_not_converted_to_none_for_chatml(tmp_path):+ csv_path = tmp_path / "train.csv"+ csv_path.write_text("instruction,input,output\nSay bye,,\n,context,answer\n")+ dataset = load_dataset("csv", data_files = str(csv_path), split = "train")++ conversations = convert_alpaca_to_chatml(dataset, batch_size = 2, num_proc = 1)["conversations"]++ assert conversations[0] == [+ {"role": "user", "content": "Say bye"},+ {"role": "assistant", "content": ""},+ ]+ assert conversations[1] == [+ {"role": "user", "content": "context"},+ {"role": "assistant", "content": "answer"},+ ]+++def test_a_blank_cell_in_a_numeric_column_still_converts_to_chatml(tmp_path):+ # The blank cell types the column float, so 1 and 3 arrive as 1.0 and 3.0.+ csv_path = tmp_path / "train.csv"+ csv_path.write_text("instruction,input,output\nA,1,x\nB,,y\nC,3,\n")+ dataset = load_dataset("csv", data_files = str(csv_path), split = "train")++ conversations = convert_alpaca_to_chatml(dataset, batch_size = 3, num_proc = 1)["conversations"]++ assert conversations[0] == [+ {"role": "user", "content": "A\n\n1.0"},+ {"role": "assistant", "content": "x"},+ ]+ assert conversations[1] == [+ {"role": "user", "content": "B"},+ {"role": "assistant", "content": "y"},+ ]+ assert conversations[2] == [+ {"role": "user", "content": "C\n\n3.0"},+ {"role": "assistant", "content": ""},+ ]+ assert all(+ isinstance(message["content"], str)+ for conversation in conversations+ for message in conversation+ )+++def test_every_cell_reaches_a_chat_template_as_text():+ # A chat template renders content directly: a non-string is a crash or a repr.+ dataset = Dataset.from_dict({"instruction": ["a"], "input": [True], "output": [7]})++ conversations = convert_alpaca_to_chatml(dataset, batch_size = 1, num_proc = 1)["conversations"]++ assert conversations[0] == [+ {"role": "user", "content": "a\n\nTrue"},+ {"role": "assistant", "content": "7"},+ ]+++def test_cell_text_maps_every_empty_spelling_to_empty_text():+ assert cell_text(None) == ""+ assert cell_text(float("nan")) == ""+ assert cell_text("") == ""++ # Present values survive, including the ones `or ""` would have dropped.+ assert cell_text("None") == "None"+ assert cell_text(0) == "0"+ assert cell_text(False) == "False"+ assert cell_text("0") == "0"+ assert cell_text(" ") == " "diff --git a/studio/backend/utils/datasets/cells.py b/studio/backend/utils/datasets/cells.pynew file mode 100644index 00000000000..17e63a07182--- /dev/null+++ b/studio/backend/utils/datasets/cells.py@@ -0,0 +1,23 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Turning one raw dataset cell into the text we are willing to train on."""+++def cell_text(value):+ """The text to train on for a single Alpaca-style dataset cell.++ A blank cell is a missing value to every loader we use, so it arrives as+ None, and a column holding numbers arrives typed, so a cell typed as 1+ arrives as 1.0. Untreated, the first trains the word None and the second+ raises on .strip(). NaN counts as blank too: Arrow normalises a blank+ numeric cell to null, but a dataset built straight from pandas need not have+ passed through Arrow.+ """+ if value is None:+ return ""+ if isinstance(value, str):+ return value+ if isinstance(value, float) and value != value:+ return ""+ return str(value)diff --git a/studio/backend/utils/datasets/chat_templates.py b/studio/backend/utils/datasets/chat_templates.pyindex afc2d3fef23..376c5fd485a 100644--- a/studio/backend/utils/datasets/chat_templates.py+++ b/studio/backend/utils/datasets/chat_templates.py@@ -5,6 +5,7 @@import warnings as python_warnings+from .cells import cell_textfrom .format_detection import detect_dataset_format, detect_multimodal_dataset, detect_custom_format_heuristicfrom .iterable import is_streaming_datasetfrom .model_mappings import MODEL_TO_TEMPLATE_MAPPER@@ -280,6 +281,7 @@ def apply_chat_template_to_dataset("input": examples.get("input", [""] * len(examples["instruction"]))[i],"output": examples["output"][i]}+ fields = {key: cell_text(value) for key, value in fields.items()}text = DEFAULT_ALPACA_TEMPLATE.format(fields["instruction"], fields["input"], fields["output"]diff --git a/studio/backend/utils/datasets/format_conversion.py b/studio/backend/utils/datasets/format_conversion.pyindex 8bafa30ac74..4dfbedc619a 100644--- a/studio/backend/utils/datasets/format_conversion.py+++ b/studio/backend/utils/datasets/format_conversion.py@@ -5,6 +5,7 @@import os+from .cells import cell_textfrom .iterable import is_streaming_datasetfrom loggers import get_logger@@ -265,6 +266,9 @@ def convert_alpaca_to_chatml(instruction = examples["instruction"][i]input_text = examples.get("input", [""] * len(examples["instruction"]))[i]output = examples["output"][i]+ instruction, input_text, output = (+ cell_text(value) for value in (instruction, input_text, output)+ )if input_text and input_text.strip():user_content = f"{instruction}\n\n{input_text}".strip()diff --git a/tests/python/test_e2e_no_torch_sandbox.py b/tests/python/test_e2e_no_torch_sandbox.pyindex 6e14bf8af09..41c4e71bc8a 100644--- a/tests/python/test_e2e_no_torch_sandbox.py+++ b/tests/python/test_e2e_no_torch_sandbox.py@@ -28,6 +28,7 @@ FORMAT_DETECTION = DATASETS_DIR / "format_detection.py"MODEL_MAPPINGS = DATASETS_DIR / "model_mappings.py"VLM_PROCESSING = DATASETS_DIR / "vlm_processing.py"ITERABLE = DATASETS_DIR / "iterable.py"+CELLS = DATASETS_DIR / "cells.py"HARDWARE_PY = HARDWARE_DIR / "hardware.py"STUDIO_VENV = Path.home() / ".unsloth" / "studio" / "unsloth_studio"@@ -277,6 +278,10 @@ class TestBeforeAfterImportChain:source = source.replace('from .format_detection import', 'from format_detection import')source = source.replace('from .model_mappings import', 'from model_mappings import')source = source.replace('from .iterable import', 'from iterable import')+ # cells.py is stdlib-only, so the exec sites import the real module+ # rather than another stub: the suite checks what actually ships.+ sys.path.insert(0, {str(CHAT_TEMPLATES.parent)!r})+ source = source.replace('from .cells import', 'from cells import')exec(source)print("OK")""")@@ -317,6 +322,7 @@ class TestBeforeAfterImportChain:DATA_COLLATORS,CHAT_TEMPLATES,ITERABLE,+ CELLS,]:if src.exists():shutil.copy2(src, pkg_dir / src.name)@@ -430,6 +436,8 @@ class TestDataclassInstantiation:source = source.replace('from .format_detection import', 'from format_detection import')source = source.replace('from .model_mappings import', 'from model_mappings import')source = source.replace('from .iterable import', 'from iterable import')+ sys.path.insert(0, {str(CHAT_TEMPLATES.parent)!r})+ source = source.replace('from .cells import', 'from cells import')exec(source, ns)assert 'Instruction' in ns['DEFAULT_ALPACA_TEMPLATE']print("OK")@@ -545,6 +553,8 @@ class TestEdgeCasesBrokenTorch:source = source.replace('from .format_detection import', 'from format_detection import')source = source.replace('from .model_mappings import', 'from model_mappings import')source = source.replace('from .iterable import', 'from iterable import')+ sys.path.insert(0, {str(CHAT_TEMPLATES.parent)!r})+ source = source.replace('from .cells import', 'from cells import')exec(source, ns)# Import succeeds -- this is the fixdiff --git a/tests/python/test_studio_import_no_torch.py b/tests/python/test_studio_import_no_torch.pyindex 9b3306df80a..503980b938e 100644--- a/tests/python/test_studio_import_no_torch.py+++ b/tests/python/test_studio_import_no_torch.py@@ -257,6 +257,10 @@ class TestChatTemplatesNoTorchVenv:source = source.replace('from .format_detection import', 'from format_detection import')source = source.replace('from .model_mappings import', 'from model_mappings import')source = source.replace('from .iterable import', 'from iterable import')+ # cells.py is stdlib-only, so the exec sites import the real module+ # rather than another stub: the suite checks what actually ships.+ sys.path.insert(0, {str(CHAT_TEMPLATES.parent)!r})+ source = source.replace('from .cells import', 'from cells import')exec(source)@@ -299,6 +303,8 @@ class TestChatTemplatesNoTorchVenv:source = source.replace('from .format_detection import', 'from format_detection import')source = source.replace('from .model_mappings import', 'from model_mappings import')source = source.replace('from .iterable import', 'from iterable import')+ sys.path.insert(0, {str(CHAT_TEMPLATES.parent)!r})+ source = source.replace('from .cells import', 'from cells import')exec(source, ns)assert 'DEFAULT_ALPACA_TEMPLATE' in ns, "DEFAULT_ALPACA_TEMPLATE not defined"@@ -391,6 +397,8 @@ class TestFormatConversionNoTorchVenv:source = open({str(FORMAT_CONVERSION)!r}, encoding = "utf-8").read()source = source.replace('from .format_detection import', 'from format_detection import')source = source.replace('from .iterable import', 'from iterable import')+ sys.path.insert(0, {str(FORMAT_CONVERSION.parent)!r})+ source = source.replace('from .cells import', 'from cells import')ns = {{'__name__': '__test__'}}exec(source, ns)@@ -448,6 +456,8 @@ class TestFormatConversionNoTorchVenv:source = open({str(FORMAT_CONVERSION)!r}, encoding = "utf-8").read()source = source.replace('from .format_detection import', 'from format_detection import')source = source.replace('from .iterable import', 'from iterable import')+ sys.path.insert(0, {str(FORMAT_CONVERSION.parent)!r})+ source = source.replace('from .cells import', 'from cells import')ns = {{'__name__': '__test__'}}exec(source, ns)