Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: train jobs targeting embed_tokens or lm_head alone, not batched (#16653)
Daniel Han68f2472aed25
diff --git a/studio/backend/core/training/multi_lora_trainer.py b/studio/backend/core/training/multi_lora_trainer.pyindex 2022a9a2bad..3bcbfb1e60b 100644--- a/studio/backend/core/training/multi_lora_trainer.py+++ b/studio/backend/core/training/multi_lora_trainer.py@@ -23,6 +23,19 @@ from loggers import get_loggerlogger = get_logger(__name__)_DEFAULT_TARGETS = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]+# Copied from unsloth.models._utils.EMBEDDING_MODULES, not imported, so the route never loads unsloth.+_EMBEDDING_TARGETS = frozenset(("embed_tokens", "lm_head"))+++def embedding_targets(target_modules) -> list[str]:+ """Targets the single job trains fully (modules_to_save); batched LoRA cannot."""+ if isinstance(target_modules, str):+ target_modules = [target_modules]+ return [+ m+ for m in target_modules or ()+ if isinstance(m, str) and m.rsplit(".", 1)[-1] in _EMBEDDING_TARGETS+ ]@dataclass@@ -904,6 +917,12 @@ def _finish_stopped(event_queue, members, save: bool, export) -> None:def _run(event_queue, stop_queue, config: dict, members: list) -> None:+ embedding = embedding_targets(config.get("target_modules"))+ if embedding:+ raise ValueError(+ f"Batched LoRA training cannot train {embedding}: they train fully on their own, so run "+ "these jobs one at a time."+ )from core.import_guards import ensure_real_packagesensure_real_packages("unsloth_zoo", "unsloth")diff --git a/studio/backend/routes/openai_fine_tuning.py b/studio/backend/routes/openai_fine_tuning.pyindex d71185adeb3..e9ccbb95385 100644--- a/studio/backend/routes/openai_fine_tuning.py+++ b/studio/backend/routes/openai_fine_tuning.py@@ -681,9 +681,12 @@ def _multi_lora_experts_available() -> bool:def _batchable(request: TrainingStartRequest) -> bool:from core.training.eval_dataset import evaluation_enabled+ from core.training.multi_lora_trainer import embedding_targetsfrom core.training.training import should_use_mlx_training_backend+return (not should_use_mlx_training_backend()+ and not embedding_targets(request.target_modules)and request.training_type == "LoRA/QLoRA"and request.use_loraand bool(request.local_datasets)diff --git a/studio/backend/tests/test_multi_lora_trainer.py b/studio/backend/tests/test_multi_lora_trainer.pyindex 45739ab185b..00c5b88e3a3 100644--- a/studio/backend/tests/test_multi_lora_trainer.py+++ b/studio/backend/tests/test_multi_lora_trainer.py@@ -993,6 +993,20 @@ def test_packing_is_refused_where_packed_documents_see_each_other(monkeypatch, tassert all(_member_events(events, m["run_id"])[-1]["type"] == "complete" for m in PACKED[:2])[email protected]("target", ["embed_tokens", "lm_head", "model.embed_tokens"])+def test_an_embedding_target_fails_instead_of_training_something_else(+ monkeypatch, tmp_path, target+):+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))+ events, model, pristine = _run(monkeypatch, MEMBERS[:2], target_modules = ["q_proj", target])+ for m in MEMBERS[:2]:+ mine = _member_events(events, m["run_id"])+ assert mine[-1]["type"] == "error" and target in mine[-1]["error"], mine[-1]+ assert not any(e.get("output_dir") for e in mine)+ assert type(model.lm_head) is type(pristine.lm_head)+ assert [e["type"] for e in events if "member_run_id" not in e][-1] == "error"++def test_an_expert_only_moe_target_trains_batched(monkeypatch, tmp_path):monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))events, _, _ = _run(diff --git a/studio/backend/tests/test_openai_fine_tuning.py b/studio/backend/tests/test_openai_fine_tuning.pyindex 129b4dcbea7..626ba9222d1 100644--- a/studio/backend/tests/test_openai_fine_tuning.py+++ b/studio/backend/tests/test_openai_fine_tuning.py@@ -1140,6 +1140,10 @@ def test_jobs_differing_only_in_member_fields_batch_in_queue_order(batching, tmp{"optim": "adamw_torch_fused"},{"gpu_ids": [0, 1]},{"start_request_id": "retry-1"},+ {"target_modules": ["q_proj", "embed_tokens"]},+ {"target_modules": ["q_proj", "lm_head"]},+ {"target_modules": ["model.embed_tokens", "v_proj"]},+ {"target_modules": ["all-linear", "lm_head"]},],)def test_an_excluded_setting_trains_alone(batching, tmp_path, fields):@@ -1153,6 +1157,16 @@ def test_an_excluded_setting_trains_alone(batching, tmp_path, fields):assert openai_fine_tuning._batch_for(plain) == [plain]+ "targets", [[], ["all-linear"], ["q_proj", "v_proj"], ["model.layers.0.self_attn.q_proj"]]+)+def test_projection_targets_still_batch(batching, tmp_path, targets):+ head = _entry(tmp_path, "a", target_modules = targets)+ other = _entry(tmp_path, "b", target_modules = targets)+ batching.extend([head, other])+ assert openai_fine_tuning._batch_for(head) == [head, other]++@pytest.mark.parametrize("fields",[