Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Stop conversation_extension crashing when the caller keeps their columns (#8373)

Vineeth Sai Varikuntla5eb891e76094
diff --git a/tests/python/test_to_sharegpt_optional_none.py b/tests/python/test_to_sharegpt_optional_none.py
index cfc7c00444f..1c0a6647f87 100644
--- a/tests/python/test_to_sharegpt_optional_none.py
+++ b/tests/python/test_to_sharegpt_optional_none.py
@@ -218,6 +218,46 @@ def test_null_cells_match_the_merged_prompt_path():
assert [r["conversations"] for r in merged] == [r["conversations"] for r in plain]
+def test_conversation_extension_with_columns_kept():
+ to_sharegpt = _load_to_sharegpt()
+ converted = to_sharegpt(
+ _alpaca(),
+ merged_prompt = "{instruction}",
+ remove_unused_columns = False,
+ conversation_extension = 3,
+ )
+
+ assert converted.column_names == ["instruction", "output", "conversations"]
+ assert len(converted[0]["conversations"]) == 6
+
+
+def test_conversation_extension_drops_its_scaffolding_columns():
+ to_sharegpt = _load_to_sharegpt()
+ for extension in (2, 3, 4):
+ converted = to_sharegpt(
+ _alpaca(),
+ merged_prompt = "{instruction}",
+ remove_unused_columns = False,
+ conversation_extension = extension,
+ )
+ numbered = [name for name in converted.column_names if name.startswith("conversations")]
+ assert numbered == ["conversations"], converted.column_names
+ assert len(converted[0]["conversations"]) == 2 * extension
+
+
+def test_conversation_extension_unchanged_when_columns_removed():
+ to_sharegpt = _load_to_sharegpt()
+ converted = to_sharegpt(
+ _alpaca(),
+ merged_prompt = "{instruction}",
+ conversation_extension = 3,
+ )
+
+ assert converted.column_names == ["conversations"]
+ assert len(converted[0]["conversations"]) == 6
+ assert all(turn["value"] for row in converted for turn in row["conversations"])
+
+
_ROWS = {"instruction": ["Sum 1 and 2"], "input": ["1 2"]}
_COLS = ["instruction", "input"]
diff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.py
index 988bbc9a324..f09f925986f 100644
--- a/unsloth/chat_templates.py
+++ b/unsloth/chat_templates.py
@@ -2362,7 +2362,8 @@ def to_sharegpt(
all_shuffled = [dataset]
for j in range(1, n_extensions+1):
shuffled = dataset.shuffle(seed = random_state+j).rename_columns({"conversations0" : f"conversations{j}"})
- all_shuffled.append(shuffled)
+ # Kept caller columns live on copy 0; repeating them makes axis=1 concat fail.
+ all_shuffled.append(shuffled.select_columns([f"conversations{j}"]))
dataset = concatenate_datasets(all_shuffled, axis = 1)
n_extensions += 1
@@ -2381,7 +2382,7 @@ def to_sharegpt(
__combine_conversations__,
batched = True,
desc = "Extending conversations",
- remove_columns = dataset.column_names if remove_unused_columns else None,
+ remove_columns = dataset.column_names if remove_unused_columns else conversation_columns,
)
return dataset