Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: allow DoRA training on Apple Silicon

Lyxot46125d0d2002
diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py
index 53295c87419..f81c773f55d 100644
--- a/studio/backend/core/training/worker.py
+++ b/studio/backend/core/training/worker.py
@@ -2568,6 +2568,39 @@ def _normalize_mlx_studio_optimizer(value):
return opt
+def _mlx_dora_peft_kwargs(config, get_peft_model):
+ """LoRA kwargs a DoRA request adds for MLX, or raise why it cannot run.
+
+ Only version skew is decided here; whether the selected modules can carry
+ DoRA depends on the loaded model, so the MLX backend refuses those itself.
+ """
+ import inspect
+
+ if not config.get("use_dora"):
+ return {}
+ try:
+ parameter = inspect.signature(get_peft_model).parameters.get("use_dora")
+ except (TypeError, ValueError):
+ parameter = None
+ # The name alone is not support: `**use_dora` collects anything, which is
+ # what the unsupporting version already does, and the positional-only and
+ # `*use_dora` kinds cannot be bound by keyword at all.
+ named = parameter is not None and parameter.kind in (
+ inspect.Parameter.POSITIONAL_OR_KEYWORD,
+ inspect.Parameter.KEYWORD_ONLY,
+ )
+ if not named:
+ raise NotImplementedError(
+ "DoRA on Apple Silicon needs an unsloth-zoo whose MLX "
+ "get_peft_model takes use_dora, and this install cannot be "
+ "confirmed to. The version that predates MLX DoRA accepts the "
+ "request and trains plain LoRA instead, so the run stops here "
+ "rather than guessing. Update unsloth-zoo, or pick a different "
+ "LoRA variant."
+ )
+ return {"use_dora": True}
+
+
def _normalize_mlx_studio_scheduler(value):
raw = str(value or "linear").strip().lower()
if raw not in _MLX_STUDIO_LR_SCHEDULERS:
@@ -2780,10 +2813,6 @@ def _run_mlx_training(event_queue, stop_queue, config):
message = "LoftQ is not supported for MLX training yet."
_send("error", error = message)
raise NotImplementedError(message)
- if config.get("use_dora"):
- message = "DoRA is not supported for MLX training yet."
- _send("error", error = message)
- raise NotImplementedError(message)
if config.get("is_embedding"):
message = "Embedding model training is not supported for MLX training yet."
_send("error", error = message)
@@ -2792,6 +2821,12 @@ def _run_mlx_training(event_queue, stop_queue, config):
message = "Continued Pretraining is not supported for MLX training yet."
_send("error", error = message)
raise NotImplementedError(message)
+ # Decided before the model loads, so version skew does not cost a download.
+ try:
+ mlx_dora_kwargs = _mlx_dora_peft_kwargs(config, FastMLXModel.get_peft_model)
+ except NotImplementedError as exc:
+ _send("error", error = str(exc))
+ raise
optim_name = _normalize_mlx_studio_optimizer(config.get("optim", "adamw_8bit"))
lr_scheduler_type = _normalize_mlx_studio_scheduler(config.get("lr_scheduler_type", "linear"))
@@ -2935,6 +2970,7 @@ def _run_mlx_training(event_queue, stop_queue, config):
],
use_gradient_checkpointing = use_grad_checkpoint,
)
+ peft_kwargs.update(mlx_dora_kwargs)
finetune_language = config.get("finetune_language_layers", True)
finetune_attention = config.get("finetune_attention_modules", True)
finetune_mlp = config.get("finetune_mlp_modules", True)
diff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.py
index 423d0bc1cd2..e210109a552 100644
--- a/studio/backend/routes/training.py
+++ b/studio/backend/routes/training.py
@@ -889,11 +889,6 @@ def _validate_training_platform(request: TrainingStartRequest) -> None:
status_code = 400,
detail = "LoftQ is not supported for MLX training yet.",
)
- if request.use_dora:
- raise HTTPException(
- status_code = 400,
- detail = "DoRA is not supported for MLX training yet.",
- )
_RESUME_DATASET_DEFAULTS = {
diff --git a/studio/backend/tests/test_mlx_training_worker_config.py b/studio/backend/tests/test_mlx_training_worker_config.py
index 4f89f0b9478..67b43a5dcaa 100644
--- a/studio/backend/tests/test_mlx_training_worker_config.py
+++ b/studio/backend/tests/test_mlx_training_worker_config.py
@@ -22,6 +22,7 @@ def _load_worker_module():
"utils.native_tls",
"utils.training_runs",
"utils.wheel_utils",
+ "utils.training_runs",
)
previous_modules = {name: sys.modules.get(name) for name in stub_names}
@@ -33,7 +34,9 @@ def _load_worker_module():
sys.modules["loggers"] = loggers
utils = types.ModuleType("utils")
- utils.__path__ = []
+ # Real package directory: an empty __path__ shadows the package and
+ # breaks the worker's own imports. Only the stubs below replace it.
+ utils.__path__ = [str(Path(__file__).resolve().parents[1] / "utils")]
sys.modules["utils"] = utils
child_stdio = types.ModuleType("utils.child_stdio")
@@ -92,6 +95,7 @@ _mlx_vlm_resized_image_layout = _worker._mlx_vlm_resized_image_layout
_copy_mlx_vlm_image_processor = _worker._copy_mlx_vlm_image_processor
_resize_mlx_vlm_image = _worker._resize_mlx_vlm_image
_adapt_for_mlx_vlm = _worker._adapt_for_mlx_vlm
+_mlx_dora_peft_kwargs = _worker._mlx_dora_peft_kwargs
def test_mlx_studio_optimizer_aliases_are_explicit():
@@ -110,6 +114,68 @@ def test_mlx_studio_rejects_unknown_scheduler():
_normalize_mlx_studio_scheduler("linear_typo")
+def test_mlx_dora_requires_the_named_use_dora_parameter():
+ # The release predating MLX DoRA swallows use_dora through this catch-all
+ # and trains plain LoRA, so accepting-the-keyword is not support.
+ def old_zoo(
+ model,
+ r = 16,
+ **kwargs,
+ ):
+ return model
+
+ def new_zoo(
+ model,
+ r = 16,
+ use_dora = False,
+ **kwargs,
+ ):
+ return model
+
+ def keyword_only_zoo(model, *, use_dora = False):
+ return model
+
+ def positional_only_zoo(
+ model,
+ use_dora = False,
+ /,
+ **kwargs,
+ ):
+ return model
+
+ def var_positional_zoo(model, *use_dora):
+ return model
+
+ def var_keyword_zoo(model, **use_dora):
+ return model
+
+ for usable in (new_zoo, keyword_only_zoo):
+ assert _mlx_dora_peft_kwargs({"use_dora": True}, usable) == {"use_dora": True}
+ with pytest.raises(NotImplementedError, match = "unsloth-zoo"):
+ _mlx_dora_peft_kwargs({"use_dora": True}, old_zoo)
+ # None of these kinds prove support: two cannot be bound by keyword, and
+ # `**use_dora` only collects it the way the unsupporting version does.
+ for unusable in (positional_only_zoo, var_positional_zoo, var_keyword_zoo):
+ with pytest.raises(NotImplementedError, match = "unsloth-zoo"):
+ _mlx_dora_peft_kwargs({"use_dora": True}, unusable)
+ # An unreadable signature (inspect.signature raises here) is not support.
+ with pytest.raises(NotImplementedError, match = "unsloth-zoo"):
+ _mlx_dora_peft_kwargs({"use_dora": True}, object())
+ # An image-bearing dataset is not proof of a vision model; a text model
+ # can still train language-only DoRA from it.
+ assert _mlx_dora_peft_kwargs(
+ {
+ "use_dora": True,
+ "is_dataset_image": True,
+ "finetune_vision_layers": True,
+ },
+ new_zoo,
+ ) == {"use_dora": True}
+ # Nothing added, nothing checked, when DoRA was not requested.
+ assert _mlx_dora_peft_kwargs({}, old_zoo) == {}
+ assert _mlx_dora_peft_kwargs({"use_dora": False}, old_zoo) == {}
+
+
def test_mlx_studio_keeps_hf_style_tokenizer_dual_purpose():
source = (Path(__file__).resolve().parents[1] / "core" / "training" / "worker.py").read_text(
encoding = "utf-8"
diff --git a/studio/backend/tests/test_training_cached_start.py b/studio/backend/tests/test_training_cached_start.py
index 4b4da5b201d..89cdbb84ec3 100644
--- a/studio/backend/tests/test_training_cached_start.py
+++ b/studio/backend/tests/test_training_cached_start.py
@@ -1259,7 +1259,6 @@ def test_streaming_rejects_cached_dataset_hints(cache_overrides):
({"is_embedding": True}, "Embedding model training"),
({"is_dataset_audio": True}, "Audio dataset training"),
({"use_loftq": True}, "LoftQ"),
- ({"use_dora": True}, "DoRA"),
],
)
def test_mlx_start_rejects_unsupported_training_config(request_overrides, expected):
@@ -1279,6 +1278,22 @@ def test_mlx_start_rejects_unsupported_training_config(request_overrides, expect
assert expected in exc_info.value.detail
+def test_mlx_start_accepts_dora():
+ # MLX trains DoRA, so the platform gate must let it through. LoftQ is
+ # asserted alongside it so a gate that stopped refusing anything at all
+ # cannot pass this test.
+ from utils.hardware import hardware
+
+ route = _load_route_module("training_route_mlx_accepts_dora")
+
+ with patch.object(hardware, "DEVICE", hardware.DeviceType.MLX):
+ route._validate_training_platform(_request(use_dora = True))
+ with pytest.raises(HTTPException) as exc_info:
+ route._validate_training_platform(_request(use_loftq = True))
+
+ assert "LoftQ" in exc_info.value.detail
+
+
def test_mlx_start_detects_hardware_before_platform_validation():
from utils.hardware import hardware
diff --git a/studio/frontend/src/features/studio/sections/lora-params-section.tsx b/studio/frontend/src/features/studio/sections/lora-params-section.tsx
index b865dea9a04..c99cbbbb427 100644
--- a/studio/frontend/src/features/studio/sections/lora-params-section.tsx
+++ b/studio/frontend/src/features/studio/sections/lora-params-section.tsx
@@ -59,6 +59,11 @@ export function LoraParamsSection(): ReactElement | null {
const [open, setOpen] = useState(true);
const isCpt = store.trainingMethod === "cpt";
const showVisionLora = store.isVisionModel && store.isDatasetImage === true;
+ // MLX refuses DoRA once vision layers are in the run. `isVisionModel` is
+ // fetched metadata that can go stale, so this only stops the combination
+ // being newly selected; the loaded model still settles an existing one.
+ const doraNeedsVisionOff =
+ deviceType === "mac" && showVisionLora && store.finetuneVisionLayers;
if (!isAdapterMethod(store.trainingMethod)) {
return null;
@@ -260,11 +265,13 @@ export function LoraParamsSection(): ReactElement | null {
store.trainingMethod,
deviceType,
);
+ const needsVisionOff =
+ option.value === "dora" && doraNeedsVisionOff;
return (
<button
key={option.value}
type="button"
- disabled={unsupportedOnMlx}
+ disabled={unsupportedOnMlx || needsVisionOff}
aria-pressed={store.loraVariant === option.value}
onClick={() => store.setLoraVariant(option.value)}
className={`flex-1 corner-squircle rounded-[14px] border px-3 py-2 text-left transition-colors cursor-pointer disabled:cursor-not-allowed disabled:opacity-60 ${selectableOptionStateClassName(store.loraVariant === option.value)}`}
@@ -273,7 +280,9 @@ export function LoraParamsSection(): ReactElement | null {
<p className="text-ui-10 text-muted-foreground">
{unsupportedOnMlx
? t("studio.params.notSupportedAppleSilicon")
- : option.desc}
+ : needsVisionOff
+ ? t("studio.params.doraNeedsVisionLayersOff")
+ : option.desc}
</p>
</button>
);
diff --git a/studio/frontend/src/features/studio/sections/use-mlx-training-config-policy.ts b/studio/frontend/src/features/studio/sections/use-mlx-training-config-policy.ts
index 14bfaf75c92..ad64cf9f014 100644
--- a/studio/frontend/src/features/studio/sections/use-mlx-training-config-policy.ts
+++ b/studio/frontend/src/features/studio/sections/use-mlx-training-config-policy.ts
@@ -15,8 +15,10 @@ export function useMlxTrainingConfigPolicy(): void {
})),
);
+ // Only loftq: MLX trains DoRA, so clearing a `dora` selection here would
+ // silently substitute plain LoRA for the run the user asked for.
useEffect(() => {
- if (isMac && (loraVariant === "loftq" || loraVariant === "dora")) {
+ if (isMac && loraVariant === "loftq") {
useTrainingConfigStore.setState({ loraVariant: "lora" });
}
}, [isMac, loraVariant]);
diff --git a/studio/frontend/src/features/training/lib/training-methods.ts b/studio/frontend/src/features/training/lib/training-methods.ts
index 033794b14ce..6272246daaa 100644
--- a/studio/frontend/src/features/training/lib/training-methods.ts
+++ b/studio/frontend/src/features/training/lib/training-methods.ts
@@ -63,11 +63,7 @@ export function isTrainingLoraVariantSupportedOnDevice(
trainingMethod === "qlora" ||
trainingMethod === "lora" ||
trainingMethod === "cpt";
- return (
- deviceType !== "mac" ||
- !usesAdapter ||
- (loraVariant !== "loftq" && loraVariant !== "dora")
- );
+ return deviceType !== "mac" || !usesAdapter || loraVariant !== "loftq";
}
export function parseBackendTrainingMethod(
diff --git a/studio/frontend/src/i18n/locales/en.ts b/studio/frontend/src/i18n/locales/en.ts
index 915435cce37..4f483933ff2 100644
--- a/studio/frontend/src/i18n/locales/en.ts
+++ b/studio/frontend/src/i18n/locales/en.ts
@@ -2125,6 +2125,7 @@ export const en = {
memoryEfficient: "Memory Efficient",
weightDecomposed: "Weight-Decomposed",
notSupportedAppleSilicon: "Not supported on Apple Silicon",
+ doraNeedsVisionLayersOff: "Turn off vision layer training to use DoRA",
optimization: "Optimization",
schedule: "Schedule",
memory: "Memory",
diff --git a/studio/frontend/tests/training-validation.test.ts b/studio/frontend/tests/training-validation.test.ts
index f76502e9eba..2b04d225e0f 100644
--- a/studio/frontend/tests/training-validation.test.ts
+++ b/studio/frontend/tests/training-validation.test.ts
@@ -263,19 +263,17 @@ test("training validation allows audio-capable vision models on MLX with image d
});
test("training validation rejects unsupported LoRA variants on MLX", () => {
- for (const loraVariant of ["loftq", "dora"] as const) {
- assert.deepEqual(
- validateTrainingConfig({ ...validConfig, loraVariant }, "mac"),
- {
- ok: false,
- errorKey: "studio.params.notSupportedAppleSilicon",
- },
- );
- assert.deepEqual(
- validateTrainingConfig({ ...validConfig, loraVariant }, "linux"),
- { ok: true, errorKey: null },
- );
- }
+ assert.deepEqual(
+ validateTrainingConfig({ ...validConfig, loraVariant: "loftq" }, "mac"),
+ {
+ ok: false,
+ errorKey: "studio.params.notSupportedAppleSilicon",
+ },
+ );
+ assert.deepEqual(
+ validateTrainingConfig({ ...validConfig, loraVariant: "loftq" }, "linux"),
+ { ok: true, errorKey: null },
+ );
assert.deepEqual(
validateTrainingConfig(
{ ...validConfig, trainingMethod: "full", loraVariant: "dora" },
@@ -285,6 +283,22 @@ test("training validation rejects unsupported LoRA variants on MLX", () => {
);
});
+test("training validation accepts DoRA on MLX under an adapter method", () => {
+ for (const trainingMethod of ["lora", "qlora", "cpt"] as const) {
+ assert.deepEqual(
+ validateTrainingConfig(
+ { ...validConfig, trainingMethod, loraVariant: "dora" },
+ "mac",
+ ),
+ // cpt is refused on MLX for its own reason, which must not be read as
+ // DoRA being refused.
+ trainingMethod === "cpt"
+ ? { ok: false, errorKey: "studio.params.notSupportedAppleSilicon" }
+ : { ok: true, errorKey: null },
+ );
+ }
+});
+
test("training validation keeps CPT and embedding training available off MLX", () => {
assert.deepEqual(
validateTrainingConfig({ ...validConfig, trainingMethod: "cpt" }, "linux"),
diff --git a/tests/studio/test_mlx_training_worker_behaviors.py b/tests/studio/test_mlx_training_worker_behaviors.py
index 0b44b0f28f9..014cb57a16c 100644
--- a/tests/studio/test_mlx_training_worker_behaviors.py
+++ b/tests/studio/test_mlx_training_worker_behaviors.py
@@ -35,6 +35,105 @@ def test_run_mlx_training_passes_token_to_from_pretrained():
assert found, "FastMLXModel.from_pretrained call not found in _run_mlx_training"
+def test_mlx_dora_decided_before_load_and_merged_into_peft_kwargs():
+ """The DoRA decision must be wired to happen before the base model is
+ fetched, and to feed the wrap call.
+
+ Dropping the merge would accept a DoRA request and silently train plain
+ LoRA; deciding after the load would make an unsupported unsloth-zoo cost
+ a multi-gigabyte download first.
+
+ This checks WIRING, not runtime behavior: the decision precedes the load,
+ takes the run config, probes the same callable the wrap invokes, and its
+ result is merged into the mapping that wrap expands without being rebound
+ in between. It cannot see through code that keeps the shape and defeats
+ the effect -- burying the merge under a false condition, unsetting the key
+ afterwards -- because the test and the code it reads live together; the
+ real-model behavior is covered by the MLX backend's own tests.
+
+ This asserts one specific shape — a `peft_kwargs.update(...)` call and a
+ `**peft_kwargs` wrap call — rather than trying to recognize every
+ equivalent formulation. Rewriting that seam another way (a merge
+ operator, explicit unpacking) is expected to fail here and to be
+ re-expressed, not worked around.
+ """
+ tree = ast.parse(WORKER.read_text(encoding = "utf-8"))
+ fn = _find_func(tree, "_run_mlx_training")
+ assert fn is not None
+
+ decided_at = None
+ decision_target = None
+ for node in ast.walk(fn):
+ if (
+ isinstance(node, ast.Assign)
+ and isinstance(node.value, ast.Call)
+ and isinstance(node.value.func, ast.Name)
+ and node.value.func.id == "_mlx_dora_peft_kwargs"
+ ):
+ decided_at = node.lineno
+ decision_target = node.targets[0].id
+ # The run's own config and the same callable the wrap invokes --
+ # probing anything else answers for something else.
+ passed = [ast.unparse(arg) for arg in node.value.args]
+ assert passed == [
+ "config",
+ "FastMLXModel.get_peft_model",
+ ], f"unexpected arguments to _mlx_dora_peft_kwargs: {passed}"
+ assert decided_at is not None, "_mlx_dora_peft_kwargs is never called"
+
+ loaded_at = min(
+ node.lineno
+ for node in ast.walk(fn)
+ if isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Attribute)
+ and node.func.attr == "from_pretrained"
+ )
+ assert decided_at < loaded_at, (
+ "DoRA support must be decided before the base model loads; "
+ f"decided at line {decided_at}, loaded at line {loaded_at}"
+ )
+
+ merge = f"peft_kwargs.update({decision_target})"
+ merged_at = [
+ node.lineno
+ for node in ast.walk(fn)
+ if isinstance(node, ast.Call) and ast.unparse(node) == merge
+ ]
+ assert merged_at, f"expected {merge}, or a DoRA request silently trains plain LoRA"
+ wraps = [
+ node
+ for node in ast.walk(fn)
+ if isinstance(node, ast.Call) and ast.unparse(node.func) == "FastMLXModel.get_peft_model"
+ ]
+ assert wraps, "FastMLXModel.get_peft_model is never called"
+ for wrap in wraps:
+ # A real ** expansion, not a string that merely looks like one.
+ assert any(
+ kw.arg is None and ast.unparse(kw.value) == "peft_kwargs" for kw in wrap.keywords
+ ), (
+ "get_peft_model must expand **peft_kwargs, or the merged DoRA "
+ f"kwargs never reach it: {ast.unparse(wrap)}"
+ )
+ first_wrap = min(wrap.lineno for wrap in wraps)
+ assert (
+ min(merged_at) < first_wrap
+ ), "peft_kwargs must be updated before get_peft_model is called"
+ # ...and not replaced in between, which would drop the merge again.
+ # A Store/Del context covers the rebinding forms a refactor realistically
+ # produces. This is a regression check over one function's line range, not
+ # a proof: string-bound names (import aliases, `case` captures) and
+ # anything sharing the merge's line are outside it.
+ rebound = [
+ node.lineno
+ for node in ast.walk(fn)
+ if isinstance(node, ast.Name)
+ and node.id == "peft_kwargs"
+ and isinstance(node.ctx, (ast.Store, ast.Del))
+ and min(merged_at) < node.lineno < first_wrap
+ ]
+ assert not rebound, f"peft_kwargs rebound at {rebound}, discarding the merge"
+
+
def test_wandb_init_strips_secret_keys():
src = WORKER.read_text(encoding = "utf-8")
assert "_wandb_sensitive" in src, "expected a sensitive-key set near wandb.init"