Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: allow DoRA training on Apple Silicon
Lyxot46125d0d2002
diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.pyindex 53295c87419..f81c773f55d 100644--- a/studio/backend/core/training/worker.py+++ b/studio/backend/core/training/worker.py@@ -2568,6 +2568,39 @@ def _normalize_mlx_studio_optimizer(value):return opt+def _mlx_dora_peft_kwargs(config, get_peft_model):+ """LoRA kwargs a DoRA request adds for MLX, or raise why it cannot run.++ Only version skew is decided here; whether the selected modules can carry+ DoRA depends on the loaded model, so the MLX backend refuses those itself.+ """+ import inspect++ if not config.get("use_dora"):+ return {}+ try:+ parameter = inspect.signature(get_peft_model).parameters.get("use_dora")+ except (TypeError, ValueError):+ parameter = None+ # The name alone is not support: `**use_dora` collects anything, which is+ # what the unsupporting version already does, and the positional-only and+ # `*use_dora` kinds cannot be bound by keyword at all.+ named = parameter is not None and parameter.kind in (+ inspect.Parameter.POSITIONAL_OR_KEYWORD,+ inspect.Parameter.KEYWORD_ONLY,+ )+ if not named:+ raise NotImplementedError(+ "DoRA on Apple Silicon needs an unsloth-zoo whose MLX "+ "get_peft_model takes use_dora, and this install cannot be "+ "confirmed to. The version that predates MLX DoRA accepts the "+ "request and trains plain LoRA instead, so the run stops here "+ "rather than guessing. Update unsloth-zoo, or pick a different "+ "LoRA variant."+ )+ return {"use_dora": True}++def _normalize_mlx_studio_scheduler(value):raw = str(value or "linear").strip().lower()if raw not in _MLX_STUDIO_LR_SCHEDULERS:@@ -2780,10 +2813,6 @@ def _run_mlx_training(event_queue, stop_queue, config):message = "LoftQ is not supported for MLX training yet."_send("error", error = message)raise NotImplementedError(message)- if config.get("use_dora"):- message = "DoRA is not supported for MLX training yet."- _send("error", error = message)- raise NotImplementedError(message)if config.get("is_embedding"):message = "Embedding model training is not supported for MLX training yet."_send("error", error = message)@@ -2792,6 +2821,12 @@ def _run_mlx_training(event_queue, stop_queue, config):message = "Continued Pretraining is not supported for MLX training yet."_send("error", error = message)raise NotImplementedError(message)+ # Decided before the model loads, so version skew does not cost a download.+ try:+ mlx_dora_kwargs = _mlx_dora_peft_kwargs(config, FastMLXModel.get_peft_model)+ except NotImplementedError as exc:+ _send("error", error = str(exc))+ raiseoptim_name = _normalize_mlx_studio_optimizer(config.get("optim", "adamw_8bit"))lr_scheduler_type = _normalize_mlx_studio_scheduler(config.get("lr_scheduler_type", "linear"))@@ -2935,6 +2970,7 @@ def _run_mlx_training(event_queue, stop_queue, config):],use_gradient_checkpointing = use_grad_checkpoint,)+ peft_kwargs.update(mlx_dora_kwargs)finetune_language = config.get("finetune_language_layers", True)finetune_attention = config.get("finetune_attention_modules", True)finetune_mlp = config.get("finetune_mlp_modules", True)diff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.pyindex 423d0bc1cd2..e210109a552 100644--- a/studio/backend/routes/training.py+++ b/studio/backend/routes/training.py@@ -889,11 +889,6 @@ def _validate_training_platform(request: TrainingStartRequest) -> None:status_code = 400,detail = "LoftQ is not supported for MLX training yet.",)- if request.use_dora:- raise HTTPException(- status_code = 400,- detail = "DoRA is not supported for MLX training yet.",- )_RESUME_DATASET_DEFAULTS = {diff --git a/studio/backend/tests/test_mlx_training_worker_config.py b/studio/backend/tests/test_mlx_training_worker_config.pyindex 4f89f0b9478..67b43a5dcaa 100644--- a/studio/backend/tests/test_mlx_training_worker_config.py+++ b/studio/backend/tests/test_mlx_training_worker_config.py@@ -22,6 +22,7 @@ def _load_worker_module():"utils.native_tls","utils.training_runs","utils.wheel_utils",+ "utils.training_runs",)previous_modules = {name: sys.modules.get(name) for name in stub_names}@@ -33,7 +34,9 @@ def _load_worker_module():sys.modules["loggers"] = loggersutils = types.ModuleType("utils")- utils.__path__ = []+ # Real package directory: an empty __path__ shadows the package and+ # breaks the worker's own imports. Only the stubs below replace it.+ utils.__path__ = [str(Path(__file__).resolve().parents[1] / "utils")]sys.modules["utils"] = utilschild_stdio = types.ModuleType("utils.child_stdio")@@ -92,6 +95,7 @@ _mlx_vlm_resized_image_layout = _worker._mlx_vlm_resized_image_layout_copy_mlx_vlm_image_processor = _worker._copy_mlx_vlm_image_processor_resize_mlx_vlm_image = _worker._resize_mlx_vlm_image_adapt_for_mlx_vlm = _worker._adapt_for_mlx_vlm+_mlx_dora_peft_kwargs = _worker._mlx_dora_peft_kwargsdef test_mlx_studio_optimizer_aliases_are_explicit():@@ -110,6 +114,68 @@ def test_mlx_studio_rejects_unknown_scheduler():_normalize_mlx_studio_scheduler("linear_typo")+def test_mlx_dora_requires_the_named_use_dora_parameter():+ # The release predating MLX DoRA swallows use_dora through this catch-all+ # and trains plain LoRA, so accepting-the-keyword is not support.+ def old_zoo(+ model,+ r = 16,+ **kwargs,+ ):+ return model++ def new_zoo(+ model,+ r = 16,+ use_dora = False,+ **kwargs,+ ):+ return model++ def keyword_only_zoo(model, *, use_dora = False):+ return model++ def positional_only_zoo(+ model,+ use_dora = False,+ /,+ **kwargs,+ ):+ return model++ def var_positional_zoo(model, *use_dora):+ return model++ def var_keyword_zoo(model, **use_dora):+ return model++ for usable in (new_zoo, keyword_only_zoo):+ assert _mlx_dora_peft_kwargs({"use_dora": True}, usable) == {"use_dora": True}+ with pytest.raises(NotImplementedError, match = "unsloth-zoo"):+ _mlx_dora_peft_kwargs({"use_dora": True}, old_zoo)+ # None of these kinds prove support: two cannot be bound by keyword, and+ # `**use_dora` only collects it the way the unsupporting version does.+ for unusable in (positional_only_zoo, var_positional_zoo, var_keyword_zoo):+ with pytest.raises(NotImplementedError, match = "unsloth-zoo"):+ _mlx_dora_peft_kwargs({"use_dora": True}, unusable)+ # An unreadable signature (inspect.signature raises here) is not support.+ with pytest.raises(NotImplementedError, match = "unsloth-zoo"):+ _mlx_dora_peft_kwargs({"use_dora": True}, object())+ # An image-bearing dataset is not proof of a vision model; a text model+ # can still train language-only DoRA from it.+ assert _mlx_dora_peft_kwargs(+ {+ "use_dora": True,+ "is_dataset_image": True,+ "finetune_vision_layers": True,+ },+ new_zoo,+ ) == {"use_dora": True}+ # Nothing added, nothing checked, when DoRA was not requested.+ assert _mlx_dora_peft_kwargs({}, old_zoo) == {}+ assert _mlx_dora_peft_kwargs({"use_dora": False}, old_zoo) == {}++def test_mlx_studio_keeps_hf_style_tokenizer_dual_purpose():source = (Path(__file__).resolve().parents[1] / "core" / "training" / "worker.py").read_text(encoding = "utf-8"diff --git a/studio/backend/tests/test_training_cached_start.py b/studio/backend/tests/test_training_cached_start.pyindex 4b4da5b201d..89cdbb84ec3 100644--- a/studio/backend/tests/test_training_cached_start.py+++ b/studio/backend/tests/test_training_cached_start.py@@ -1259,7 +1259,6 @@ def test_streaming_rejects_cached_dataset_hints(cache_overrides):({"is_embedding": True}, "Embedding model training"),({"is_dataset_audio": True}, "Audio dataset training"),({"use_loftq": True}, "LoftQ"),- ({"use_dora": True}, "DoRA"),],)def test_mlx_start_rejects_unsupported_training_config(request_overrides, expected):@@ -1279,6 +1278,22 @@ def test_mlx_start_rejects_unsupported_training_config(request_overrides, expectassert expected in exc_info.value.detail+def test_mlx_start_accepts_dora():+ # MLX trains DoRA, so the platform gate must let it through. LoftQ is+ # asserted alongside it so a gate that stopped refusing anything at all+ # cannot pass this test.+ from utils.hardware import hardware++ route = _load_route_module("training_route_mlx_accepts_dora")++ with patch.object(hardware, "DEVICE", hardware.DeviceType.MLX):+ route._validate_training_platform(_request(use_dora = True))+ with pytest.raises(HTTPException) as exc_info:+ route._validate_training_platform(_request(use_loftq = True))++ assert "LoftQ" in exc_info.value.detail++def test_mlx_start_detects_hardware_before_platform_validation():from utils.hardware import hardwarediff --git a/studio/frontend/src/features/studio/sections/lora-params-section.tsx b/studio/frontend/src/features/studio/sections/lora-params-section.tsxindex b865dea9a04..c99cbbbb427 100644--- a/studio/frontend/src/features/studio/sections/lora-params-section.tsx+++ b/studio/frontend/src/features/studio/sections/lora-params-section.tsx@@ -59,6 +59,11 @@ export function LoraParamsSection(): ReactElement | null {const [open, setOpen] = useState(true);const isCpt = store.trainingMethod === "cpt";const showVisionLora = store.isVisionModel && store.isDatasetImage === true;+ // MLX refuses DoRA once vision layers are in the run. `isVisionModel` is+ // fetched metadata that can go stale, so this only stops the combination+ // being newly selected; the loaded model still settles an existing one.+ const doraNeedsVisionOff =+ deviceType === "mac" && showVisionLora && store.finetuneVisionLayers;if (!isAdapterMethod(store.trainingMethod)) {return null;@@ -260,11 +265,13 @@ export function LoraParamsSection(): ReactElement | null {store.trainingMethod,deviceType,);+ const needsVisionOff =+ option.value === "dora" && doraNeedsVisionOff;return (<buttonkey={option.value}type="button"- disabled={unsupportedOnMlx}+ disabled={unsupportedOnMlx || needsVisionOff}aria-pressed={store.loraVariant === option.value}onClick={() => store.setLoraVariant(option.value)}className={`flex-1 corner-squircle rounded-[14px] border px-3 py-2 text-left transition-colors cursor-pointer disabled:cursor-not-allowed disabled:opacity-60 ${selectableOptionStateClassName(store.loraVariant === option.value)}`}@@ -273,7 +280,9 @@ export function LoraParamsSection(): ReactElement | null {<p className="text-ui-10 text-muted-foreground">{unsupportedOnMlx? t("studio.params.notSupportedAppleSilicon")- : option.desc}+ : needsVisionOff+ ? t("studio.params.doraNeedsVisionLayersOff")+ : option.desc}</p></button>);diff --git a/studio/frontend/src/features/studio/sections/use-mlx-training-config-policy.ts b/studio/frontend/src/features/studio/sections/use-mlx-training-config-policy.tsindex 14bfaf75c92..ad64cf9f014 100644--- a/studio/frontend/src/features/studio/sections/use-mlx-training-config-policy.ts+++ b/studio/frontend/src/features/studio/sections/use-mlx-training-config-policy.ts@@ -15,8 +15,10 @@ export function useMlxTrainingConfigPolicy(): void {})),);+ // Only loftq: MLX trains DoRA, so clearing a `dora` selection here would+ // silently substitute plain LoRA for the run the user asked for.useEffect(() => {- if (isMac && (loraVariant === "loftq" || loraVariant === "dora")) {+ if (isMac && loraVariant === "loftq") {useTrainingConfigStore.setState({ loraVariant: "lora" });}}, [isMac, loraVariant]);diff --git a/studio/frontend/src/features/training/lib/training-methods.ts b/studio/frontend/src/features/training/lib/training-methods.tsindex 033794b14ce..6272246daaa 100644--- a/studio/frontend/src/features/training/lib/training-methods.ts+++ b/studio/frontend/src/features/training/lib/training-methods.ts@@ -63,11 +63,7 @@ export function isTrainingLoraVariantSupportedOnDevice(trainingMethod === "qlora" ||trainingMethod === "lora" ||trainingMethod === "cpt";- return (- deviceType !== "mac" ||- !usesAdapter ||- (loraVariant !== "loftq" && loraVariant !== "dora")- );+ return deviceType !== "mac" || !usesAdapter || loraVariant !== "loftq";}export function parseBackendTrainingMethod(diff --git a/studio/frontend/src/i18n/locales/en.ts b/studio/frontend/src/i18n/locales/en.tsindex 915435cce37..4f483933ff2 100644--- a/studio/frontend/src/i18n/locales/en.ts+++ b/studio/frontend/src/i18n/locales/en.ts@@ -2125,6 +2125,7 @@ export const en = {memoryEfficient: "Memory Efficient",weightDecomposed: "Weight-Decomposed",notSupportedAppleSilicon: "Not supported on Apple Silicon",+ doraNeedsVisionLayersOff: "Turn off vision layer training to use DoRA",optimization: "Optimization",schedule: "Schedule",memory: "Memory",diff --git a/studio/frontend/tests/training-validation.test.ts b/studio/frontend/tests/training-validation.test.tsindex f76502e9eba..2b04d225e0f 100644--- a/studio/frontend/tests/training-validation.test.ts+++ b/studio/frontend/tests/training-validation.test.ts@@ -263,19 +263,17 @@ test("training validation allows audio-capable vision models on MLX with image d});test("training validation rejects unsupported LoRA variants on MLX", () => {- for (const loraVariant of ["loftq", "dora"] as const) {- assert.deepEqual(- validateTrainingConfig({ ...validConfig, loraVariant }, "mac"),- {- ok: false,- errorKey: "studio.params.notSupportedAppleSilicon",- },- );- assert.deepEqual(- validateTrainingConfig({ ...validConfig, loraVariant }, "linux"),- { ok: true, errorKey: null },- );- }+ assert.deepEqual(+ validateTrainingConfig({ ...validConfig, loraVariant: "loftq" }, "mac"),+ {+ ok: false,+ errorKey: "studio.params.notSupportedAppleSilicon",+ },+ );+ assert.deepEqual(+ validateTrainingConfig({ ...validConfig, loraVariant: "loftq" }, "linux"),+ { ok: true, errorKey: null },+ );assert.deepEqual(validateTrainingConfig({ ...validConfig, trainingMethod: "full", loraVariant: "dora" },@@ -285,6 +283,22 @@ test("training validation rejects unsupported LoRA variants on MLX", () => {);});+test("training validation accepts DoRA on MLX under an adapter method", () => {+ for (const trainingMethod of ["lora", "qlora", "cpt"] as const) {+ assert.deepEqual(+ validateTrainingConfig(+ { ...validConfig, trainingMethod, loraVariant: "dora" },+ "mac",+ ),+ // cpt is refused on MLX for its own reason, which must not be read as+ // DoRA being refused.+ trainingMethod === "cpt"+ ? { ok: false, errorKey: "studio.params.notSupportedAppleSilicon" }+ : { ok: true, errorKey: null },+ );+ }+});+test("training validation keeps CPT and embedding training available off MLX", () => {assert.deepEqual(validateTrainingConfig({ ...validConfig, trainingMethod: "cpt" }, "linux"),diff --git a/tests/studio/test_mlx_training_worker_behaviors.py b/tests/studio/test_mlx_training_worker_behaviors.pyindex 0b44b0f28f9..014cb57a16c 100644--- a/tests/studio/test_mlx_training_worker_behaviors.py+++ b/tests/studio/test_mlx_training_worker_behaviors.py@@ -35,6 +35,105 @@ def test_run_mlx_training_passes_token_to_from_pretrained():assert found, "FastMLXModel.from_pretrained call not found in _run_mlx_training"+def test_mlx_dora_decided_before_load_and_merged_into_peft_kwargs():+ """The DoRA decision must be wired to happen before the base model is+ fetched, and to feed the wrap call.++ Dropping the merge would accept a DoRA request and silently train plain+ LoRA; deciding after the load would make an unsupported unsloth-zoo cost+ a multi-gigabyte download first.++ This checks WIRING, not runtime behavior: the decision precedes the load,+ takes the run config, probes the same callable the wrap invokes, and its+ result is merged into the mapping that wrap expands without being rebound+ in between. It cannot see through code that keeps the shape and defeats+ the effect -- burying the merge under a false condition, unsetting the key+ afterwards -- because the test and the code it reads live together; the+ real-model behavior is covered by the MLX backend's own tests.++ This asserts one specific shape — a `peft_kwargs.update(...)` call and a+ `**peft_kwargs` wrap call — rather than trying to recognize every+ equivalent formulation. Rewriting that seam another way (a merge+ operator, explicit unpacking) is expected to fail here and to be+ re-expressed, not worked around.+ """+ tree = ast.parse(WORKER.read_text(encoding = "utf-8"))+ fn = _find_func(tree, "_run_mlx_training")+ assert fn is not None++ decided_at = None+ decision_target = None+ for node in ast.walk(fn):+ if (+ isinstance(node, ast.Assign)+ and isinstance(node.value, ast.Call)+ and isinstance(node.value.func, ast.Name)+ and node.value.func.id == "_mlx_dora_peft_kwargs"+ ):+ decided_at = node.lineno+ decision_target = node.targets[0].id+ # The run's own config and the same callable the wrap invokes --+ # probing anything else answers for something else.+ passed = [ast.unparse(arg) for arg in node.value.args]+ assert passed == [+ "config",+ "FastMLXModel.get_peft_model",+ ], f"unexpected arguments to _mlx_dora_peft_kwargs: {passed}"+ assert decided_at is not None, "_mlx_dora_peft_kwargs is never called"++ loaded_at = min(+ node.lineno+ for node in ast.walk(fn)+ if isinstance(node, ast.Call)+ and isinstance(node.func, ast.Attribute)+ and node.func.attr == "from_pretrained"+ )+ assert decided_at < loaded_at, (+ "DoRA support must be decided before the base model loads; "+ f"decided at line {decided_at}, loaded at line {loaded_at}"+ )++ merge = f"peft_kwargs.update({decision_target})"+ merged_at = [+ node.lineno+ for node in ast.walk(fn)+ if isinstance(node, ast.Call) and ast.unparse(node) == merge+ ]+ assert merged_at, f"expected {merge}, or a DoRA request silently trains plain LoRA"+ wraps = [+ node+ for node in ast.walk(fn)+ if isinstance(node, ast.Call) and ast.unparse(node.func) == "FastMLXModel.get_peft_model"+ ]+ assert wraps, "FastMLXModel.get_peft_model is never called"+ for wrap in wraps:+ # A real ** expansion, not a string that merely looks like one.+ assert any(+ kw.arg is None and ast.unparse(kw.value) == "peft_kwargs" for kw in wrap.keywords+ ), (+ "get_peft_model must expand **peft_kwargs, or the merged DoRA "+ f"kwargs never reach it: {ast.unparse(wrap)}"+ )+ first_wrap = min(wrap.lineno for wrap in wraps)+ assert (+ min(merged_at) < first_wrap+ ), "peft_kwargs must be updated before get_peft_model is called"+ # ...and not replaced in between, which would drop the merge again.+ # A Store/Del context covers the rebinding forms a refactor realistically+ # produces. This is a regression check over one function's line range, not+ # a proof: string-bound names (import aliases, `case` captures) and+ # anything sharing the merge's line are outside it.+ rebound = [+ node.lineno+ for node in ast.walk(fn)+ if isinstance(node, ast.Name)+ and node.id == "peft_kwargs"+ and isinstance(node.ctx, (ast.Store, ast.Del))+ and min(merged_at) < node.lineno < first_wrap+ ]+ assert not rebound, f"peft_kwargs rebound at {rebound}, discarding the merge"++def test_wandb_init_strips_secret_keys():src = WORKER.read_text(encoding = "utf-8")assert "_wandb_sensitive" in src, "expected a sensitive-key set near wandb.init"