Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop leftover dataset columns becoming the system prompt (#12485)
Nilayefe3860ce2bf
diff --git a/studio/backend/hub/utils/dataset_format.py b/studio/backend/hub/utils/dataset_format.pyindex c93a342d871..2934c79cf00 100644--- a/studio/backend/hub/utils/dataset_format.py+++ b/studio/backend/hub/utils/dataset_format.py@@ -183,9 +183,59 @@ def detect_custom_format_heuristic(dataset):score += 10return score+ context_words = {+ "input",+ "passage",+ "text",+ "document",+ "article",+ "contract",+ "evidence",+ "story",+ "paragraph",+ "definition",+ "background",+ "knowledge",+ "choices",+ "options",+ }++ meta_tokens = {"id", "ids", "idx", "type", "category", "label", "title", "tag"}++ def name_tokens(col_name):+ separated_name = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", "_", col_name)+ return set(re.findall(r"[a-z]+", separated_name.lower()))++ def is_non_metadata_column(col_name):+ return not meta_tokens & name_tokens(col_name)++ def is_prompt_content_column(col_name):+ return is_non_metadata_column(col_name) and not isinstance(+ sample.get(col_name), (bool, int, float)+ )++ def is_context_column(col_name):+ tokens = name_tokens(col_name)+ return (+ any(token in context_words or token[:-1] in context_words for token in tokens)+ and is_prompt_content_column(col_name)+ and not has_keyword(col_name, assistant_words)+ )+content_columns = [col for col in all_columns if not is_metadata(col)]+ system_named = [+ col+ for col in content_columns+ if has_keyword(col, ["system"])+ and is_prompt_content_column(col)+ and not has_keyword(col, assistant_words)+ ]assistant_potential = [col for col in content_columns if has_keyword(col, assistant_words)]- user_potential = [col for col in content_columns if has_keyword(col, user_words)]+ user_potential = [+ col+ for col in content_columns+ if col not in system_named and is_non_metadata_column(col) and has_keyword(col, user_words)+ ]assistant_candidates = [(col, score)for col in assistant_potential@@ -212,7 +262,10 @@ def detect_custom_format_heuristic(dataset):shadowed_potential = [colfor col in content_columns- if col not in user_potential and has_keyword(col, user_words, apply_shadowing = False)+ if col not in system_named+ and col not in user_potential+ and is_non_metadata_column(col)+ and has_keyword(col, user_words, apply_shadowing = False)]for col in shadowed_potential:if col == assistant_col:@@ -230,31 +283,61 @@ def detect_custom_format_heuristic(dataset):user_col = Noneremaining_columns = [col for col in content_columns if col not in mapping]- system_col = None- for col in remaining_columns:- if has_keyword(col, system_words):- mapping[col] = "system"- system_col = col- break+ if user_col is not None and is_context_column(user_col):+ for col in remaining_columns:+ if (+ has_keyword(col, user_words_high_priority)+ and isinstance(sample.get(col), str)+ and not meta_tokens & name_tokens(col)+ and not is_context_column(col)+ and not has_keyword(col, assistant_words)+ ):+ del mapping[user_col]+ mapping[col] = "user"+ remaining_columns = [c for c in remaining_columns if c != col] + [user_col]+ user_col = col+ break++ non_task_system_words = [word for word in system_words if word != "task"]+ system_tiers = [+ lambda col: col in system_named,+ lambda col: has_keyword(col, non_task_system_words) and is_prompt_content_column(col),+ lambda col: user_col is not None and is_context_column(col),+ lambda col: has_keyword(col, system_words) and is_prompt_content_column(col),+ ]+ system_col = next(+ (col for tier in system_tiers for col in remaining_columns if tier(col)), None+ )if system_col:+ mapping[system_col] = "system"remaining_columns = [col for col in remaining_columns if col != system_col]if remaining_columns:remaining_col = remaining_columns[0]- if not has_keyword(remaining_col, user_words + assistant_words):- mapping[remaining_col] = "system"- elif user_col is None:+ if (+ user_col is None+ and has_keyword(remaining_col, user_words)+ and not has_keyword(remaining_col, assistant_words + system_words)+ and is_non_metadata_column(remaining_col)+ ):mapping[remaining_col] = "user"- elif not has_keyword(remaining_col, assistant_words):- mapping[remaining_col] = "system"has_user = any(role == "user" for role in mapping.values())has_assistant = any(role == "assistant" for role in mapping.values())if not has_user:- for col in remaining_columns:- if col not in mapping:+ for col in remaining_columns[1:]:+ if (+ col not in mapping+ and is_prompt_content_column(col)+ and not has_keyword(col, assistant_words + system_words)+ ):mapping[col] = "user"has_user = Truebreak+ if system_col is None:+ for col in remaining_columns:+ if col not in mapping and is_context_column(col):+ mapping[col] = "system"+ breakreturn mapping if has_user and has_assistant else Nonediff --git a/studio/backend/tests/test_dataset_heuristic_role_keywords.py b/studio/backend/tests/test_dataset_heuristic_role_keywords.pyindex 5604ebab39e..f31bb356353 100644--- a/studio/backend/tests/test_dataset_heuristic_role_keywords.py+++ b/studio/backend/tests/test_dataset_heuristic_role_keywords.py@@ -32,7 +32,7 @@ _CASES = [),({"id": "1", "title": "t", "context": _LONG, "question": _MID, "answers": _MID},- {"question": "user", "context": "system", "title": "system", "answers": "assistant"},+ {"question": "user", "context": "system", "answers": "assistant"},),({"instruction": _MID, "context": _LONG, "response": _MID, "category": "qa"},@@ -110,9 +110,13 @@ _ASSISTANT_LEFTOVER_CASES = [# for a manual mapping instead of silently training the system prompt as the user turn._NO_USER_COLUMN_ROWS = [{"system": _MID, "output": _MID},+ {"system_prompt": _MID, "output": _MID},+ {"system_prompt": _MID, "context_id": "c1", "output": _MID},{"persona": _MID, "reply": _MID},{"role": _MID, "response": _MID},{"template": _MID, "output": _MID},+ {"question_type": "causal", "answer": _MID},+ {"answer": _MID, "explanation": _MID},]@@ -158,3 +162,113 @@ _ANSWER_LEFTOVER_CASES = [@pytest.mark.parametrize("row, expected", _ANSWER_LEFTOVER_CASES)def test_answer_column_is_not_mapped_to_the_system_prompt(heuristic, row, expected):assert heuristic([row]) == expected+++_LEFTOVER_CASES = [+ (+ {"type": "GSM_SV", "query": _MID, "original_question": _MID, "response": _LONG},+ {"query": "user", "response": "assistant"},+ ),+ (+ {"id": "1", "input": _MID, "output": _LONG, "source": "cf", "license": "cc-by-4.0"},+ {"input": "user", "output": "assistant"},+ ),+ (+ {"prompt": _MID, "response": _LONG, "helpfulness": 3, "correctness": 3},+ {"prompt": "user", "response": "assistant"},+ ),+ (+ {"question": _MID, "distractor3": "viruses", "correct_answer": _MID, "support": _LONG},+ {"question": "user", "correct_answer": "assistant"},+ ),+ (+ {"Question": _MID, "Complex_CoT": _LONG, "Response": _MID},+ {"Question": "user", "Response": "assistant"},+ ),+ (+ {"task_id": "HumanEval/0", "prompt": _LONG, "canonical_solution": _MID, "test": _LONG},+ {"prompt": "user", "canonical_solution": "assistant"},+ ),+ (+ {"Instruction": "Translate this.", "Input": "It is sunny.", "Output": "Il fait beau."},+ {"Instruction": "user", "Input": "system", "Output": "assistant"},+ ),+ (+ {"instruction": "Translate this.", "input": "It is sunny.", "response": "Il fait beau."},+ {"instruction": "user", "input": "system", "response": "assistant"},+ ),+ (+ {"question": _MID, "answer": "true", "passage": _LONG},+ {"question": "user", "answer": "assistant", "passage": "system"},+ ),+]++[email protected]("heuristic", _HEURISTICS)[email protected]("row, expected", _LEFTOVER_CASES)+def test_leftover_column_is_not_mapped_to_the_system_prompt(heuristic, row, expected):+ assert heuristic([row]) == expected+++_CONTEXT_CASES = [+ (+ {"id": "f.1", "system_prompt": _LONG, "question": _MID, "response": _LONG},+ {"question": "user", "system_prompt": "system", "response": "assistant"},+ ),+ (+ {"task_id": "t1", "system": "Be brief.", "prompt": _MID, "response": _MID},+ {"prompt": "user", "system": "system", "response": "assistant"},+ ),+ (+ {"index": 0, "question": _MID, "text": _LONG, "answer": "Yes"},+ {"question": "user", "text": "system", "answer": "assistant"},+ ),+ (+ {"article": _LONG, "question": _MID, "options": ["a", "b"], "answer": 1},+ {"article": "system", "question": "user", "answer": "assistant"},+ ),+ (+ {"task_name": "task001_quoref", "definition": _LONG, "inputs": _MID, "targets": _MID},+ {"definition": "system", "inputs": "user", "targets": "assistant"},+ ),+ (+ {"prompt": _MID, "response": _LONG, "input_tokens": 12, "passage_id": "p1"},+ {"prompt": "user", "response": "assistant"},+ ),+ (+ {"question": _MID, "answer": _MID, "input_ids": [1, 2], "background": _LONG},+ {"question": "user", "answer": "assistant", "background": "system"},+ ),+ (+ {"text": _LONG, "question_type": "causal", "answer": "yes"},+ {"text": "user", "answer": "assistant"},+ ),+]++[email protected]("heuristic", _HEURISTICS)[email protected]("row, expected", _CONTEXT_CASES)+def test_system_and_context_columns_keep_the_system_prompt(heuristic, row, expected):+ assert heuristic([row]) == expected++[email protected]("heuristic", _HEURISTICS)+def test_identifier_column_is_not_the_user_turn(heuristic):+ assert heuristic([{"task_id": "HumanEval/0", "canonical_solution": _MID}]) is None+++_SYSTEM_METADATA_CASES = [+ {"question": _MID, "answer": _MID, "system_id": "s1"},+ {"question": _MID, "answer": _MID, "context_id": "c1"},+ {"question": _MID, "answer": _MID, "systemId": "s1"},+ {"question": _MID, "answer": _MID, "contextId": "c1"},+ {"question": _MID, "answer": _MID, "systemID": "s1"},+ {"question": _MID, "answer": _MID, "contextID": "c1"},+ {"question": _MID, "answer": _MID, "context_length": 4096},+]++[email protected]("heuristic", _HEURISTICS)[email protected]("row", _SYSTEM_METADATA_CASES)+def test_system_metadata_is_not_mapped_to_the_system_prompt(heuristic, row):+ assert heuristic([row]) == {"question": "user", "answer": "assistant"}diff --git a/studio/backend/utils/datasets/format_detection.py b/studio/backend/utils/datasets/format_detection.pyindex 2ff0a3adbed..12d2a884ced 100644--- a/studio/backend/utils/datasets/format_detection.py+++ b/studio/backend/utils/datasets/format_detection.py@@ -355,10 +355,60 @@ def detect_custom_format_heuristic(dataset):return score+ context_words = {+ "input",+ "passage",+ "text",+ "document",+ "article",+ "contract",+ "evidence",+ "story",+ "paragraph",+ "definition",+ "background",+ "knowledge",+ "choices",+ "options",+ }++ meta_tokens = {"id", "ids", "idx", "type", "category", "label", "title", "tag"}++ def name_tokens(col_name):+ separated_name = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", "_", col_name)+ return set(re.findall(r"[a-z]+", separated_name.lower()))++ def is_non_metadata_column(col_name):+ return not meta_tokens & name_tokens(col_name)++ def is_prompt_content_column(col_name):+ return is_non_metadata_column(col_name) and not isinstance(+ sample.get(col_name), (bool, int, float)+ )++ def is_context_column(col_name):+ tokens = name_tokens(col_name)+ return (+ any(token in context_words or token[:-1] in context_words for token in tokens)+ and is_prompt_content_column(col_name)+ and not has_keyword(col_name, assistant_words)+ )+content_columns = [col for col in all_columns if not is_metadata(col)]+ system_named = [+ col+ for col in content_columns+ if has_keyword(col, ["system"])+ and is_prompt_content_column(col)+ and not has_keyword(col, assistant_words)+ ]assistant_potential = [col for col in content_columns if has_keyword(col, assistant_words)]- user_potential = [col for col in content_columns if has_keyword(col, user_words)]+ user_potential = [+ col+ for col in content_columns+ if col not in system_named and is_non_metadata_column(col) and has_keyword(col, user_words)+ ]assistant_candidates = []for col in assistant_potential:@@ -388,7 +438,10 @@ def detect_custom_format_heuristic(dataset):shadowed_potential = [colfor col in content_columns- if col not in user_potential and has_keyword(col, user_words, apply_shadowing = False)+ if col not in system_named+ and col not in user_potential+ and is_non_metadata_column(col)+ and has_keyword(col, user_words, apply_shadowing = False)]for col in shadowed_potential:if col == assistant_col:@@ -408,37 +461,66 @@ def detect_custom_format_heuristic(dataset):remaining_columns = [col for col in content_columns if col not in mapping]- system_col = None- for col in remaining_columns:- if has_keyword(col, system_words):- mapping[col] = "system"- system_col = col- break+ if user_col is not None and is_context_column(user_col):+ for col in remaining_columns:+ if (+ has_keyword(col, user_words_high_priority)+ and isinstance(sample.get(col), str)+ and not meta_tokens & name_tokens(col)+ and not is_context_column(col)+ and not has_keyword(col, assistant_words)+ ):+ del mapping[user_col]+ mapping[col] = "user"+ remaining_columns = [c for c in remaining_columns if c != col] + [user_col]+ user_col = col+ break+ non_task_system_words = [word for word in system_words if word != "task"]+ system_tiers = [+ lambda col: col in system_named,+ lambda col: has_keyword(col, non_task_system_words) and is_prompt_content_column(col),+ lambda col: user_col is not None and is_context_column(col),+ lambda col: has_keyword(col, system_words) and is_prompt_content_column(col),+ ]+ system_col = next(+ (col for tier in system_tiers for col in remaining_columns if tier(col)), None+ )if system_col:+ mapping[system_col] = "system"remaining_columns = [col for col in remaining_columns if col != system_col]if len(remaining_columns) >= 1:remaining_col = remaining_columns[0]- # No strong keyword match: decide by what is missing.- if not has_keyword(remaining_col, user_words + assistant_words):- mapping[remaining_col] = "system"- elif user_col is None:+ if (+ user_col is None+ and has_keyword(remaining_col, user_words)+ and not has_keyword(remaining_col, assistant_words + system_words)+ and is_non_metadata_column(remaining_col)+ ):mapping[remaining_col] = "user"- elif not has_keyword(remaining_col, assistant_words):- mapping[remaining_col] = "system"has_user = any(role == "user" for role in mapping.values())has_assistant = any(role == "assistant" for role in mapping.values())if not has_user and len(remaining_columns) > 0:- for col in remaining_columns:- if col not in mapping:+ for col in remaining_columns[1:]:+ if (+ col not in mapping+ and is_prompt_content_column(col)+ and not has_keyword(col, assistant_words + system_words)+ ):mapping[col] = "user"has_user = Truebreak+ if system_col is None:+ for col in remaining_columns:+ if col not in mapping and is_context_column(col):+ mapping[col] = "system"+ break+if has_user and has_assistant:return mapping