Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: train jobs targeting embed_tokens or lm_head alone, not batched (#16653)

Daniel Han68f2472aed25
diff --git a/studio/backend/core/training/multi_lora_trainer.py b/studio/backend/core/training/multi_lora_trainer.py
index 2022a9a2bad..3bcbfb1e60b 100644
--- a/studio/backend/core/training/multi_lora_trainer.py
+++ b/studio/backend/core/training/multi_lora_trainer.py
@@ -23,6 +23,19 @@ from loggers import get_logger
logger = get_logger(__name__)
_DEFAULT_TARGETS = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]
+# Copied from unsloth.models._utils.EMBEDDING_MODULES, not imported, so the route never loads unsloth.
+_EMBEDDING_TARGETS = frozenset(("embed_tokens", "lm_head"))
+
+
+def embedding_targets(target_modules) -> list[str]:
+ """Targets the single job trains fully (modules_to_save); batched LoRA cannot."""
+ if isinstance(target_modules, str):
+ target_modules = [target_modules]
+ return [
+ m
+ for m in target_modules or ()
+ if isinstance(m, str) and m.rsplit(".", 1)[-1] in _EMBEDDING_TARGETS
+ ]
@dataclass
@@ -904,6 +917,12 @@ def _finish_stopped(event_queue, members, save: bool, export) -> None:
def _run(event_queue, stop_queue, config: dict, members: list) -> None:
+ embedding = embedding_targets(config.get("target_modules"))
+ if embedding:
+ raise ValueError(
+ f"Batched LoRA training cannot train {embedding}: they train fully on their own, so run "
+ "these jobs one at a time."
+ )
from core.import_guards import ensure_real_packages
ensure_real_packages("unsloth_zoo", "unsloth")
diff --git a/studio/backend/routes/openai_fine_tuning.py b/studio/backend/routes/openai_fine_tuning.py
index d71185adeb3..e9ccbb95385 100644
--- a/studio/backend/routes/openai_fine_tuning.py
+++ b/studio/backend/routes/openai_fine_tuning.py
@@ -681,9 +681,12 @@ def _multi_lora_experts_available() -> bool:
def _batchable(request: TrainingStartRequest) -> bool:
from core.training.eval_dataset import evaluation_enabled
+ from core.training.multi_lora_trainer import embedding_targets
from core.training.training import should_use_mlx_training_backend
+
return (
not should_use_mlx_training_backend()
+ and not embedding_targets(request.target_modules)
and request.training_type == "LoRA/QLoRA"
and request.use_lora
and bool(request.local_datasets)
diff --git a/studio/backend/tests/test_multi_lora_trainer.py b/studio/backend/tests/test_multi_lora_trainer.py
index 45739ab185b..00c5b88e3a3 100644
--- a/studio/backend/tests/test_multi_lora_trainer.py
+++ b/studio/backend/tests/test_multi_lora_trainer.py
@@ -993,6 +993,20 @@ def test_packing_is_refused_where_packed_documents_see_each_other(monkeypatch, t
assert all(_member_events(events, m["run_id"])[-1]["type"] == "complete" for m in PACKED[:2])
[email protected]("target", ["embed_tokens", "lm_head", "model.embed_tokens"])
+def test_an_embedding_target_fails_instead_of_training_something_else(
+ monkeypatch, tmp_path, target
+):
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
+ events, model, pristine = _run(monkeypatch, MEMBERS[:2], target_modules = ["q_proj", target])
+ for m in MEMBERS[:2]:
+ mine = _member_events(events, m["run_id"])
+ assert mine[-1]["type"] == "error" and target in mine[-1]["error"], mine[-1]
+ assert not any(e.get("output_dir") for e in mine)
+ assert type(model.lm_head) is type(pristine.lm_head)
+ assert [e["type"] for e in events if "member_run_id" not in e][-1] == "error"
+
+
def test_an_expert_only_moe_target_trains_batched(monkeypatch, tmp_path):
monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
events, _, _ = _run(
diff --git a/studio/backend/tests/test_openai_fine_tuning.py b/studio/backend/tests/test_openai_fine_tuning.py
index 129b4dcbea7..626ba9222d1 100644
--- a/studio/backend/tests/test_openai_fine_tuning.py
+++ b/studio/backend/tests/test_openai_fine_tuning.py
@@ -1140,6 +1140,10 @@ def test_jobs_differing_only_in_member_fields_batch_in_queue_order(batching, tmp
{"optim": "adamw_torch_fused"},
{"gpu_ids": [0, 1]},
{"start_request_id": "retry-1"},
+ {"target_modules": ["q_proj", "embed_tokens"]},
+ {"target_modules": ["q_proj", "lm_head"]},
+ {"target_modules": ["model.embed_tokens", "v_proj"]},
+ {"target_modules": ["all-linear", "lm_head"]},
],
)
def test_an_excluded_setting_trains_alone(batching, tmp_path, fields):
@@ -1153,6 +1157,16 @@ def test_an_excluded_setting_trains_alone(batching, tmp_path, fields):
assert openai_fine_tuning._batch_for(plain) == [plain]
+ "targets", [[], ["all-linear"], ["q_proj", "v_proj"], ["model.layers.0.self_attn.q_proj"]]
+)
+def test_projection_targets_still_batch(batching, tmp_path, targets):
+ head = _entry(tmp_path, "a", target_modules = targets)
+ other = _entry(tmp_path, "b", target_modules = targets)
+ batching.extend([head, other])
+ assert openai_fine_tuning._batch_for(head) == [head, other]
+
+
@pytest.mark.parametrize(
"fields",
[