Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: read uploaded CSV cells as written for embedding training (#14014)
Nilayff68ecc75789
diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.pyindex bd9b7adbd31..5bf50dbcb82 100644--- a/studio/backend/core/training/worker.py+++ b/studio/backend/core/training/worker.py@@ -5472,6 +5472,15 @@ def _create_embedding_progress_callback(return _EmbeddingProgressCallback()+def _csv_number(cell):+ from utils.datasets.cells import _NA_CELLS++ text = (cell or "").strip()+ if text.lower() in ("true", "false"):+ return float(text.lower() == "true")+ return None if text in _NA_CELLS else float(text)++def _run_embedding_training(event_queue: Any, stop_queue: Any, config: dict) -> None:"""Self-contained embedding model training pipeline. Uses FastSentenceTransformer +SentenceTransformerTrainer + MultipleNegativesRankingLoss, separate from UnslothTrainer's@@ -5662,6 +5671,7 @@ def _run_embedding_training(event_queue: Any, stop_queue: Any, config: dict) ->config["_dataset_loaded_from_exact_snapshot"] = Falsehf_dataset = str(config.get("hf_dataset") or "").strip()local_datasets = config.get("local_datasets") or []+ csv_text_columns: list[str] = []def _load_local_embedding_dataset(dataset_paths: list[str]):from utils.paths import dataset_files_in_dir@@ -5694,7 +5704,29 @@ def _run_embedding_training(event_queue: Any, stop_queue: Any, config: dict) ->loader = "parquet"else:raise ValueError(f"Unsupported local dataset format: {all_files[0]}")- return load_dataset(loader, data_files = all_files, split = "train")+ from datasets import Features, Value+ from utils.datasets.cells import csv_as_text_kwargs, typed_csv_columns++ dataset = load_dataset(+ loader,+ data_files = all_files,+ split = "train",+ **csv_as_text_kwargs(all_files),+ )+ if loader != "csv":+ return dataset+ labels = typed_csv_columns(dataset) & {"label", "labels", "score", "scores"}+ for column in labels:+ dataset = dataset.map(+ lambda cells: {column: [_csv_number(cell) for cell in cells]},+ input_columns = column,+ batched = True,+ features = Features({**dataset.features, column: Value("float64")}),+ )+ csv_text_columns[:] = [+ column for column in dataset.column_names if column not in labels+ ]+ return datasetif hf_dataset:dataset = _load_embedding_hf_dataset(@@ -5749,6 +5781,25 @@ def _run_embedding_training(event_queue: Any, stop_queue: Any, config: dict) ->end = slice_end if slice_end is not None else len(dataset)dataset = dataset.select(range(start, min(end + 1, len(dataset))))+ if csv_text_columns:+ kept = dataset.filter(+ lambda *columns: [None not in cells for cells in zip(*columns)],+ input_columns = csv_text_columns,+ batched = True,+ )+ if len(dataset) and not len(kept):+ raise ValueError(f"All {len(dataset):,} rows have an empty text cell.")+ if len(kept) < len(dataset):+ event_queue.put(+ {+ "type": "warning",+ "message": f"Dropped {len(dataset) - len(kept):,} of {len(dataset):,} "+ "rows because a text cell was empty.",+ "ts": time.time(),+ }+ )+ dataset = kept+logger.info(f"Embedding dataset loaded: {len(dataset)} samples")except Exception as e:event_queue.put(diff --git a/studio/backend/tests/test_embedding_csv_cells_as_written.py b/studio/backend/tests/test_embedding_csv_cells_as_written.pynew file mode 100644index 00000000000..6910d3085f1--- /dev/null+++ b/studio/backend/tests/test_embedding_csv_cells_as_written.py@@ -0,0 +1,197 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++from __future__ import annotations++import sys+from pathlib import Path++_BACKEND = Path(__file__).resolve().parent.parent+if str(_BACKEND) not in sys.path:+ sys.path.insert(0, str(_BACKEND))++import importlib # noqa: E402+import types # noqa: E402+from unittest.mock import MagicMock # noqa: E402++import pytest # noqa: E402++_STUBBED: list[str] = []+++def _stub_if_missing(name, attrs):+ if name in sys.modules:+ return+ try:+ importlib.import_module(name)+ return+ except Exception: # noqa: BLE001 - stub unusable imports+ pass+ _STUBBED.append(name)+ mod = types.ModuleType(name)+ mod.__spec__ = None+ for attr in attrs:+ setattr(mod, attr, MagicMock())+ sys.modules[name] = mod+ parent, _, child = name.rpartition(".")+ if parent and parent in sys.modules:+ setattr(sys.modules[parent], child, mod)+++_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))+_stub_if_missing("unsloth.chat_templates", ("get_chat_template",))+_stub_if_missing("trl", ("SFTTrainer", "SFTConfig"))++from core.training import trainer as _trainer # noqa: E402,F401+from core.training import worker # noqa: E402++for _name in reversed(_STUBBED):+ sys.modules.pop(_name, None)+++def _module(name, **attrs):+ mod = types.ModuleType(name)+ mod.__spec__ = None+ mod.__dict__.update(attrs)+ return mod+++def run_embedding(monkeypatch, tmp_path):+ trained = {}++ class Trainer:+ def __init__(self, **kwargs):+ trained.update(kwargs)++ def remove_callback(self, callback):+ pass++ def train(self, **kwargs):+ pass++ monkeypatch.setitem(+ sys.modules,+ "unsloth",+ _module(+ "unsloth",+ FastSentenceTransformer = MagicMock(),+ is_bfloat16_supported = lambda: False,+ ),+ )+ monkeypatch.setitem(+ sys.modules,+ "sentence_transformers",+ _module(+ "sentence_transformers",+ SentenceTransformerTrainer = Trainer,+ SentenceTransformerTrainingArguments = MagicMock(),+ ),+ )+ monkeypatch.setitem(+ sys.modules,+ "sentence_transformers.losses",+ _module("sentence_transformers.losses", MultipleNegativesRankingLoss = MagicMock()),+ )+ monkeypatch.setitem(+ sys.modules,+ "sentence_transformers.training_args",+ _module("sentence_transformers.training_args", BatchSamplers = MagicMock()),+ )+ monkeypatch.setattr(worker, "_model_load_security_error", lambda *a, **k: None)+ monkeypatch.setattr(worker, "_start_worker_stop_poller", lambda *a, **k: None)+ monkeypatch.setattr(worker, "_emit_resource_provenance", lambda *a, **k: None)++ def run(+ csv_text,+ error = None,+ **config,+ ):+ path = tmp_path / "pairs.csv"+ path.write_text(csv_text)+ events = []+ worker._run_embedding_training(+ types.SimpleNamespace(put = events.append),+ None,+ {+ "model_name": "unsloth/all-MiniLM-L6-v2",+ "training_type": "Full Finetuning",+ "local_datasets": [str(path)],+ "output_dir": "embedding-csv-cells",+ "max_steps": 1,+ **config,+ },+ )+ errors = [event["error"] for event in events if event["type"] == "error"]+ assert errors == ([error] if error else [])+ warnings = [event["message"] for event in events if event["type"] == "warning"]+ return trained.get("train_dataset"), warnings++ return run+++def test_uploaded_csv_trains_embedding_pairs_as_written(run_embedding):+ dataset, warnings = run_embedding(+ "query,answer\n"+ "Zip code of Holtsville NY?,00501\n"+ "Price for item 7?,NA\n"+ "What does the API return on a miss?,None\n"+ "Say nothing.,\n"+ "Zip code of Schenectady NY?,12345\n"+ )++ assert dataset.to_list() == [+ {"query": "Zip code of Holtsville NY?", "answer": "00501"},+ {"query": "Price for item 7?", "answer": "NA"},+ {"query": "What does the API return on a miss?", "answer": "None"},+ {"query": "Zip code of Schenectady NY?", "answer": "12345"},+ ]+ assert warnings == ["Dropped 1 of 5 rows because a text cell was empty."]+++def test_embedding_csv_score_column_still_reaches_the_trainer_as_numbers(run_embedding):+ dataset, warnings = run_embedding(+ "sentence1,sentence2,score\nA cat sits.,A cat is sitting.,0.9\nA dog runs.,Rain falls.,0\n"+ )++ assert dataset.to_list() == [+ {"sentence1": "A cat sits.", "sentence2": "A cat is sitting.", "score": 0.9},+ {"sentence1": "A dog runs.", "sentence2": "Rain falls.", "score": 0.0},+ ]+ assert warnings == []+++def test_embedding_csv_label_columns_parse_like_pandas(run_embedding):+ dataset, _ = run_embedding("s1,s2,label,score\naa,bb,True, 0.5\ncc,dd,False,0.25 \n")++ assert dataset.to_list() == [+ {"s1": "aa", "s2": "bb", "label": 1.0, "score": 0.5},+ {"s1": "cc", "s2": "dd", "label": 0.0, "score": 0.25},+ ]+++def test_embedding_csv_with_no_text_left_fails_instead_of_training_nothing(run_embedding):+ run_embedding(+ "s1,s2,\naa,bb,\ncc,dd,\n",+ error = "Failed to load dataset: All 2 rows have an empty text cell.",+ )+++def test_embedding_csv_slice_counts_rows_of_the_file(run_embedding):+ dataset, warnings = run_embedding(+ "q,a\nq0,a0\nq1,\nq2,a2\nq3,a3\nq4,a4\n",+ dataset_slice_start = 1,+ dataset_slice_end = 3,+ )++ assert dataset.to_list() == [{"q": "q2", "a": "a2"}, {"q": "q3", "a": "a3"}]+ assert warnings == ["Dropped 1 of 3 rows because a text cell was empty."]+++def test_embedding_csv_score_na_marker_outside_the_slice_still_trains(run_embedding):+ dataset, _ = run_embedding(+ "s1,s2,score\naa,bb,0.5\ncc,dd,NA\n",+ dataset_slice_end = 0,+ )++ assert dataset.to_list() == [{"s1": "aa", "s2": "bb", "score": 0.5}]