Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop leftover dataset columns becoming the system prompt (#12485)

Nilayefe3860ce2bf
diff --git a/studio/backend/hub/utils/dataset_format.py b/studio/backend/hub/utils/dataset_format.py
index c93a342d871..2934c79cf00 100644
--- a/studio/backend/hub/utils/dataset_format.py
+++ b/studio/backend/hub/utils/dataset_format.py
@@ -183,9 +183,59 @@ def detect_custom_format_heuristic(dataset):
score += 10
return score
+ context_words = {
+ "input",
+ "passage",
+ "text",
+ "document",
+ "article",
+ "contract",
+ "evidence",
+ "story",
+ "paragraph",
+ "definition",
+ "background",
+ "knowledge",
+ "choices",
+ "options",
+ }
+
+ meta_tokens = {"id", "ids", "idx", "type", "category", "label", "title", "tag"}
+
+ def name_tokens(col_name):
+ separated_name = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", "_", col_name)
+ return set(re.findall(r"[a-z]+", separated_name.lower()))
+
+ def is_non_metadata_column(col_name):
+ return not meta_tokens & name_tokens(col_name)
+
+ def is_prompt_content_column(col_name):
+ return is_non_metadata_column(col_name) and not isinstance(
+ sample.get(col_name), (bool, int, float)
+ )
+
+ def is_context_column(col_name):
+ tokens = name_tokens(col_name)
+ return (
+ any(token in context_words or token[:-1] in context_words for token in tokens)
+ and is_prompt_content_column(col_name)
+ and not has_keyword(col_name, assistant_words)
+ )
+
content_columns = [col for col in all_columns if not is_metadata(col)]
+ system_named = [
+ col
+ for col in content_columns
+ if has_keyword(col, ["system"])
+ and is_prompt_content_column(col)
+ and not has_keyword(col, assistant_words)
+ ]
assistant_potential = [col for col in content_columns if has_keyword(col, assistant_words)]
- user_potential = [col for col in content_columns if has_keyword(col, user_words)]
+ user_potential = [
+ col
+ for col in content_columns
+ if col not in system_named and is_non_metadata_column(col) and has_keyword(col, user_words)
+ ]
assistant_candidates = [
(col, score)
for col in assistant_potential
@@ -212,7 +262,10 @@ def detect_custom_format_heuristic(dataset):
shadowed_potential = [
col
for col in content_columns
- if col not in user_potential and has_keyword(col, user_words, apply_shadowing = False)
+ if col not in system_named
+ and col not in user_potential
+ and is_non_metadata_column(col)
+ and has_keyword(col, user_words, apply_shadowing = False)
]
for col in shadowed_potential:
if col == assistant_col:
@@ -230,31 +283,61 @@ def detect_custom_format_heuristic(dataset):
user_col = None
remaining_columns = [col for col in content_columns if col not in mapping]
- system_col = None
- for col in remaining_columns:
- if has_keyword(col, system_words):
- mapping[col] = "system"
- system_col = col
- break
+ if user_col is not None and is_context_column(user_col):
+ for col in remaining_columns:
+ if (
+ has_keyword(col, user_words_high_priority)
+ and isinstance(sample.get(col), str)
+ and not meta_tokens & name_tokens(col)
+ and not is_context_column(col)
+ and not has_keyword(col, assistant_words)
+ ):
+ del mapping[user_col]
+ mapping[col] = "user"
+ remaining_columns = [c for c in remaining_columns if c != col] + [user_col]
+ user_col = col
+ break
+
+ non_task_system_words = [word for word in system_words if word != "task"]
+ system_tiers = [
+ lambda col: col in system_named,
+ lambda col: has_keyword(col, non_task_system_words) and is_prompt_content_column(col),
+ lambda col: user_col is not None and is_context_column(col),
+ lambda col: has_keyword(col, system_words) and is_prompt_content_column(col),
+ ]
+ system_col = next(
+ (col for tier in system_tiers for col in remaining_columns if tier(col)), None
+ )
if system_col:
+ mapping[system_col] = "system"
remaining_columns = [col for col in remaining_columns if col != system_col]
if remaining_columns:
remaining_col = remaining_columns[0]
- if not has_keyword(remaining_col, user_words + assistant_words):
- mapping[remaining_col] = "system"
- elif user_col is None:
+ if (
+ user_col is None
+ and has_keyword(remaining_col, user_words)
+ and not has_keyword(remaining_col, assistant_words + system_words)
+ and is_non_metadata_column(remaining_col)
+ ):
mapping[remaining_col] = "user"
- elif not has_keyword(remaining_col, assistant_words):
- mapping[remaining_col] = "system"
has_user = any(role == "user" for role in mapping.values())
has_assistant = any(role == "assistant" for role in mapping.values())
if not has_user:
- for col in remaining_columns:
- if col not in mapping:
+ for col in remaining_columns[1:]:
+ if (
+ col not in mapping
+ and is_prompt_content_column(col)
+ and not has_keyword(col, assistant_words + system_words)
+ ):
mapping[col] = "user"
has_user = True
break
+ if system_col is None:
+ for col in remaining_columns:
+ if col not in mapping and is_context_column(col):
+ mapping[col] = "system"
+ break
return mapping if has_user and has_assistant else None
diff --git a/studio/backend/tests/test_dataset_heuristic_role_keywords.py b/studio/backend/tests/test_dataset_heuristic_role_keywords.py
index 5604ebab39e..f31bb356353 100644
--- a/studio/backend/tests/test_dataset_heuristic_role_keywords.py
+++ b/studio/backend/tests/test_dataset_heuristic_role_keywords.py
@@ -32,7 +32,7 @@ _CASES = [
),
(
{"id": "1", "title": "t", "context": _LONG, "question": _MID, "answers": _MID},
- {"question": "user", "context": "system", "title": "system", "answers": "assistant"},
+ {"question": "user", "context": "system", "answers": "assistant"},
),
(
{"instruction": _MID, "context": _LONG, "response": _MID, "category": "qa"},
@@ -110,9 +110,13 @@ _ASSISTANT_LEFTOVER_CASES = [
# for a manual mapping instead of silently training the system prompt as the user turn.
_NO_USER_COLUMN_ROWS = [
{"system": _MID, "output": _MID},
+ {"system_prompt": _MID, "output": _MID},
+ {"system_prompt": _MID, "context_id": "c1", "output": _MID},
{"persona": _MID, "reply": _MID},
{"role": _MID, "response": _MID},
{"template": _MID, "output": _MID},
+ {"question_type": "causal", "answer": _MID},
+ {"answer": _MID, "explanation": _MID},
]
@@ -158,3 +162,113 @@ _ANSWER_LEFTOVER_CASES = [
@pytest.mark.parametrize("row, expected", _ANSWER_LEFTOVER_CASES)
def test_answer_column_is_not_mapped_to_the_system_prompt(heuristic, row, expected):
assert heuristic([row]) == expected
+
+
+_LEFTOVER_CASES = [
+ (
+ {"type": "GSM_SV", "query": _MID, "original_question": _MID, "response": _LONG},
+ {"query": "user", "response": "assistant"},
+ ),
+ (
+ {"id": "1", "input": _MID, "output": _LONG, "source": "cf", "license": "cc-by-4.0"},
+ {"input": "user", "output": "assistant"},
+ ),
+ (
+ {"prompt": _MID, "response": _LONG, "helpfulness": 3, "correctness": 3},
+ {"prompt": "user", "response": "assistant"},
+ ),
+ (
+ {"question": _MID, "distractor3": "viruses", "correct_answer": _MID, "support": _LONG},
+ {"question": "user", "correct_answer": "assistant"},
+ ),
+ (
+ {"Question": _MID, "Complex_CoT": _LONG, "Response": _MID},
+ {"Question": "user", "Response": "assistant"},
+ ),
+ (
+ {"task_id": "HumanEval/0", "prompt": _LONG, "canonical_solution": _MID, "test": _LONG},
+ {"prompt": "user", "canonical_solution": "assistant"},
+ ),
+ (
+ {"Instruction": "Translate this.", "Input": "It is sunny.", "Output": "Il fait beau."},
+ {"Instruction": "user", "Input": "system", "Output": "assistant"},
+ ),
+ (
+ {"instruction": "Translate this.", "input": "It is sunny.", "response": "Il fait beau."},
+ {"instruction": "user", "input": "system", "response": "assistant"},
+ ),
+ (
+ {"question": _MID, "answer": "true", "passage": _LONG},
+ {"question": "user", "answer": "assistant", "passage": "system"},
+ ),
+]
+
+
[email protected]("heuristic", _HEURISTICS)
[email protected]("row, expected", _LEFTOVER_CASES)
+def test_leftover_column_is_not_mapped_to_the_system_prompt(heuristic, row, expected):
+ assert heuristic([row]) == expected
+
+
+_CONTEXT_CASES = [
+ (
+ {"id": "f.1", "system_prompt": _LONG, "question": _MID, "response": _LONG},
+ {"question": "user", "system_prompt": "system", "response": "assistant"},
+ ),
+ (
+ {"task_id": "t1", "system": "Be brief.", "prompt": _MID, "response": _MID},
+ {"prompt": "user", "system": "system", "response": "assistant"},
+ ),
+ (
+ {"index": 0, "question": _MID, "text": _LONG, "answer": "Yes"},
+ {"question": "user", "text": "system", "answer": "assistant"},
+ ),
+ (
+ {"article": _LONG, "question": _MID, "options": ["a", "b"], "answer": 1},
+ {"article": "system", "question": "user", "answer": "assistant"},
+ ),
+ (
+ {"task_name": "task001_quoref", "definition": _LONG, "inputs": _MID, "targets": _MID},
+ {"definition": "system", "inputs": "user", "targets": "assistant"},
+ ),
+ (
+ {"prompt": _MID, "response": _LONG, "input_tokens": 12, "passage_id": "p1"},
+ {"prompt": "user", "response": "assistant"},
+ ),
+ (
+ {"question": _MID, "answer": _MID, "input_ids": [1, 2], "background": _LONG},
+ {"question": "user", "answer": "assistant", "background": "system"},
+ ),
+ (
+ {"text": _LONG, "question_type": "causal", "answer": "yes"},
+ {"text": "user", "answer": "assistant"},
+ ),
+]
+
+
[email protected]("heuristic", _HEURISTICS)
[email protected]("row, expected", _CONTEXT_CASES)
+def test_system_and_context_columns_keep_the_system_prompt(heuristic, row, expected):
+ assert heuristic([row]) == expected
+
+
[email protected]("heuristic", _HEURISTICS)
+def test_identifier_column_is_not_the_user_turn(heuristic):
+ assert heuristic([{"task_id": "HumanEval/0", "canonical_solution": _MID}]) is None
+
+
+_SYSTEM_METADATA_CASES = [
+ {"question": _MID, "answer": _MID, "system_id": "s1"},
+ {"question": _MID, "answer": _MID, "context_id": "c1"},
+ {"question": _MID, "answer": _MID, "systemId": "s1"},
+ {"question": _MID, "answer": _MID, "contextId": "c1"},
+ {"question": _MID, "answer": _MID, "systemID": "s1"},
+ {"question": _MID, "answer": _MID, "contextID": "c1"},
+ {"question": _MID, "answer": _MID, "context_length": 4096},
+]
+
+
[email protected]("heuristic", _HEURISTICS)
[email protected]("row", _SYSTEM_METADATA_CASES)
+def test_system_metadata_is_not_mapped_to_the_system_prompt(heuristic, row):
+ assert heuristic([row]) == {"question": "user", "answer": "assistant"}
diff --git a/studio/backend/utils/datasets/format_detection.py b/studio/backend/utils/datasets/format_detection.py
index 2ff0a3adbed..12d2a884ced 100644
--- a/studio/backend/utils/datasets/format_detection.py
+++ b/studio/backend/utils/datasets/format_detection.py
@@ -355,10 +355,60 @@ def detect_custom_format_heuristic(dataset):
return score
+ context_words = {
+ "input",
+ "passage",
+ "text",
+ "document",
+ "article",
+ "contract",
+ "evidence",
+ "story",
+ "paragraph",
+ "definition",
+ "background",
+ "knowledge",
+ "choices",
+ "options",
+ }
+
+ meta_tokens = {"id", "ids", "idx", "type", "category", "label", "title", "tag"}
+
+ def name_tokens(col_name):
+ separated_name = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", "_", col_name)
+ return set(re.findall(r"[a-z]+", separated_name.lower()))
+
+ def is_non_metadata_column(col_name):
+ return not meta_tokens & name_tokens(col_name)
+
+ def is_prompt_content_column(col_name):
+ return is_non_metadata_column(col_name) and not isinstance(
+ sample.get(col_name), (bool, int, float)
+ )
+
+ def is_context_column(col_name):
+ tokens = name_tokens(col_name)
+ return (
+ any(token in context_words or token[:-1] in context_words for token in tokens)
+ and is_prompt_content_column(col_name)
+ and not has_keyword(col_name, assistant_words)
+ )
+
content_columns = [col for col in all_columns if not is_metadata(col)]
+ system_named = [
+ col
+ for col in content_columns
+ if has_keyword(col, ["system"])
+ and is_prompt_content_column(col)
+ and not has_keyword(col, assistant_words)
+ ]
assistant_potential = [col for col in content_columns if has_keyword(col, assistant_words)]
- user_potential = [col for col in content_columns if has_keyword(col, user_words)]
+ user_potential = [
+ col
+ for col in content_columns
+ if col not in system_named and is_non_metadata_column(col) and has_keyword(col, user_words)
+ ]
assistant_candidates = []
for col in assistant_potential:
@@ -388,7 +438,10 @@ def detect_custom_format_heuristic(dataset):
shadowed_potential = [
col
for col in content_columns
- if col not in user_potential and has_keyword(col, user_words, apply_shadowing = False)
+ if col not in system_named
+ and col not in user_potential
+ and is_non_metadata_column(col)
+ and has_keyword(col, user_words, apply_shadowing = False)
]
for col in shadowed_potential:
if col == assistant_col:
@@ -408,37 +461,66 @@ def detect_custom_format_heuristic(dataset):
remaining_columns = [col for col in content_columns if col not in mapping]
- system_col = None
- for col in remaining_columns:
- if has_keyword(col, system_words):
- mapping[col] = "system"
- system_col = col
- break
+ if user_col is not None and is_context_column(user_col):
+ for col in remaining_columns:
+ if (
+ has_keyword(col, user_words_high_priority)
+ and isinstance(sample.get(col), str)
+ and not meta_tokens & name_tokens(col)
+ and not is_context_column(col)
+ and not has_keyword(col, assistant_words)
+ ):
+ del mapping[user_col]
+ mapping[col] = "user"
+ remaining_columns = [c for c in remaining_columns if c != col] + [user_col]
+ user_col = col
+ break
+ non_task_system_words = [word for word in system_words if word != "task"]
+ system_tiers = [
+ lambda col: col in system_named,
+ lambda col: has_keyword(col, non_task_system_words) and is_prompt_content_column(col),
+ lambda col: user_col is not None and is_context_column(col),
+ lambda col: has_keyword(col, system_words) and is_prompt_content_column(col),
+ ]
+ system_col = next(
+ (col for tier in system_tiers for col in remaining_columns if tier(col)), None
+ )
if system_col:
+ mapping[system_col] = "system"
remaining_columns = [col for col in remaining_columns if col != system_col]
if len(remaining_columns) >= 1:
remaining_col = remaining_columns[0]
- # No strong keyword match: decide by what is missing.
- if not has_keyword(remaining_col, user_words + assistant_words):
- mapping[remaining_col] = "system"
- elif user_col is None:
+ if (
+ user_col is None
+ and has_keyword(remaining_col, user_words)
+ and not has_keyword(remaining_col, assistant_words + system_words)
+ and is_non_metadata_column(remaining_col)
+ ):
mapping[remaining_col] = "user"
- elif not has_keyword(remaining_col, assistant_words):
- mapping[remaining_col] = "system"
has_user = any(role == "user" for role in mapping.values())
has_assistant = any(role == "assistant" for role in mapping.values())
if not has_user and len(remaining_columns) > 0:
- for col in remaining_columns:
- if col not in mapping:
+ for col in remaining_columns[1:]:
+ if (
+ col not in mapping
+ and is_prompt_content_column(col)
+ and not has_keyword(col, assistant_words + system_words)
+ ):
mapping[col] = "user"
has_user = True
break
+ if system_col is None:
+ for col in remaining_columns:
+ if col not in mapping and is_context_column(col):
+ mapping[col] = "system"
+ break
+
if has_user and has_assistant:
return mapping