Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
MLX: Clef Omni in Unsloth Studio, with audio in the Decision API and decision training (#13538)
Long Yixingdaab3bcd80cb
diff --git a/studio/backend/core/systemone/catalog.py b/studio/backend/core/systemone/catalog.pyindex e8305d0db5f..4f4d9598385 100644--- a/studio/backend/core/systemone/catalog.py+++ b/studio/backend/core/systemone/catalog.py@@ -31,6 +31,8 @@ class Checkpoint:revision: str | None = None# Display name where the frontend has none of its own.label: str | None = None+ # A Hub entry whose model hears audio; a local folder says so in its config.+ audio: bool = False@propertydef is_local(self) -> bool:@@ -83,6 +85,18 @@ CHECKPOINTS = {54_976_000_000,"clef",),+ Checkpoint(+ "clef-omni",+ "mlx-community/clef-omni-8bit",+ None,+ "Cloudflare Clef Omni on Qwen3-Omni 30B (8-bit, MLX only): text, images and audio, "+ "needs Apple Silicon with 48 GB of memory or more.",+ 35_044_686_208,+ "clef",+ "mlx",+ label = "Clef Omni",+ audio = True,+ ),_gguf("kev-0.8b","Kev 0.8B",diff --git a/studio/backend/core/systemone/laya_runtime.py b/studio/backend/core/systemone/laya_runtime.pyindex a421f7b0343..50e761a427f 100644--- a/studio/backend/core/systemone/laya_runtime.py+++ b/studio/backend/core/systemone/laya_runtime.py@@ -160,8 +160,11 @@ def _engine_available() -> bool:def _mlx_target(checkpoint: Checkpoint) -> Checkpoint | None:"""What unsloth-zoo's MLX engine serves in process on Apple Silicon: a merged Clef's own folder, or the source repo of a GGUF entry."""- if _is_mlx(checkpoint) or not _engine_available():+ if not _engine_available():return None+ if _is_mlx(checkpoint):+ # Only a catalog entry is its own MLX form; a target has none further.+ return checkpoint if CHECKPOINTS.get(checkpoint.name) == checkpoint else Noneif checkpoint.layout == "clef":if checkpoint.is_local:from utils.models.model_config import clef_folder_kind@@ -180,18 +183,18 @@ def _mlx_target(checkpoint: Checkpoint) -> Checkpoint | None:)-def _mlx_reads_images(checkpoint: Checkpoint) -> bool:- """Whether the MLX form of an entry reads images: a Clef with its vision tower and processor, on an unsloth-zoo that reads them."""+def _mlx_reads(checkpoint: Checkpoint, media: str) -> bool:+ """Whether the MLX form of an entry reads `media` ("images" or "audio"): a Clef with that tower and its processor, on an unsloth-zoo that reads it."""if checkpoint.layout != "clef" or _mlx_target(checkpoint) is None:return Falsetry:from unsloth_zoo.mlx.decision import ClefModelexcept Exception:return False- if not getattr(ClefModel, "takes_images", False):+ if not getattr(ClefModel, f"takes_{media}", False):return Falseif not checkpoint.is_local:- return True+ return media == "images" or checkpoint.audioimport jsonfolder = Path(checkpoint.source).expanduser()@@ -199,14 +202,24 @@ def _mlx_reads_images(checkpoint: Checkpoint) -> bool:config = json.loads((folder / "config.json").read_text(encoding = "utf-8"))except (OSError, ValueError):return False- return (- isinstance(config, dict)- and config.get("vision_config") is not None- and any(- (folder / name).is_file()- for name in ("processor_config.json", "preprocessor_config.json")- )+ if not isinstance(config, dict):+ return False+ # Qwen3-Omni keeps its towers under the thinker, Phi-4-multimodal under `embd_layer`; Nemotron Omni hears through a sound tower.+ parts = (config, config.get("thinker_config"), config.get("embd_layer"))+ towers = (+ ("vision_config", "image_embd_layer")+ if media == "images"+ else ("audio_config", "sound_config", "audio_embd_layer"))+ return any(+ isinstance(part, dict) and part.get(tower) is not None for part in parts for tower in towers+ ) and any(+ (folder / name).is_file() for name in ("processor_config.json", "preprocessor_config.json")+ )+++def _mlx_reads_images(checkpoint: Checkpoint) -> bool:+ return _mlx_reads(checkpoint, "images")def _mlx_repos(companion) -> tuple:@@ -423,12 +436,14 @@ def select(questions = None,preference: str | None = None,state_images: bool = False,+ audio = None,) -> tuple[Checkpoint, str | None]:"""(what serves this request, why Auto did not pick llama.cpp); raises for what nothing here can serve.Auto takes llama.cpp when the entry has a GGUF and Studio's llama-server serves decisions, else PyTorch;on Apple Silicon it answers through unsloth-zoo's MLX engine first.- Images are read by llama.cpp, and by MLX for a Clef. Laya is always served by PyTorch, a GGUF entry otherwise by llama.cpp.+ Images are read by llama.cpp, and by MLX for a Clef. Audio is heard only by MLX, for a Clef with an audio tower.+ Laya is always served by PyTorch, a GGUF entry otherwise by llama.cpp."""from utils.systemone_settings import get_backend@@ -436,6 +451,29 @@ def select(# Only route-checked images reach MLX: state images would be decoded unvalidated in this process.unread = bool(state_images or (images and not _mlx_reads_images(checkpoint)))+ if audio:+ # No other runtime hears, so nothing else may answer as if the clips were not sent.+ if not _mlx_reads(checkpoint, "audio"):+ raise Unavailable(+ 400,+ "api_usage_error",+ f"{checkpoint.name} does not hear audio here; audio needs a Clef with an audio tower, served by MLX on Apple Silicon.",+ )+ if (preference or get_backend()) not in ("auto", MLX):+ raise Unavailable(+ 400,+ "api_usage_error",+ "Audio is served only by MLX; set the Decision API runtime to Auto or MLX.",+ )+ return _select_mlx(checkpoint, unread), None+ if _is_mlx(checkpoint):+ if (preference or get_backend()) not in ("auto", MLX):+ raise Unavailable(+ 400,+ "api_usage_error",+ f"{checkpoint.name} is served only by MLX; set the Decision API runtime to Auto or MLX.",+ )+ return _select_mlx(checkpoint, unread), Noneif (preference or get_backend()) == MLX and checkpoint.layout in ("clef", GGUF):return _select_mlx(checkpoint, unread), Nonemlx = _mlx_choice(@@ -501,6 +539,10 @@ def _select_mlx(checkpoint: Checkpoint, unread) -> Checkpoint:return targetelse:reason = f"{checkpoint.name} has no MLX form."+ if _is_mlx(checkpoint):+ raise Unavailable(+ 400, "api_usage_error", f"{checkpoint.name} is served only by MLX. {reason}"+ )raise Unavailable(400, "api_usage_error", f"{reason} Set the Decision API runtime to Auto.")@@ -575,9 +617,15 @@ def native_ready(checkpoint) -> bool:def input_modalities(checkpoint) -> list[str]:- """What this entry reads here: images too when llama.cpp serves it with a vision projector, or MLX a Clef that has its tower."""- if mlx_ready(checkpoint) and _mlx_reads_images(checkpoint):- return ["text", "image"]+ """What this entry reads here: images too when llama.cpp serves it with a vision projector, or MLX a Clef that has its tower; audio when MLX serves a Clef that hears."""+ if mlx_ready(checkpoint):+ media = [+ kind+ for kind, media in (("image", "images"), ("audio", "audio"))+ if _mlx_reads(checkpoint, media)+ ]+ if media:+ return ["text", *media]if not native_ready(checkpoint):return ["text"]return ["text", "image"] if _native_reads_images(_native_target(checkpoint)) else ["text"]@@ -940,6 +988,7 @@ class _EngineAgent:state,questions: dict[str, dict[str, Any]],images = None,+ audio = None,) -> dict[str, Any]:from unsloth_zoo.mlx.decision import DecisionRequestError, DecisionUnsupportedError@@ -977,11 +1026,9 @@ class _EngineAgent:f"The state is longer than the {limit} tokens this model reads beside a question. Shorten it.",)try:- return (- self.model.answer(state, questions, images)- if images- else self.model.answer(state, questions)- )+ # Only what was sent is passed, so an unsloth-zoo that takes less still answers text.+ media = (images, audio) if audio else (images,) if images else ()+ return self.model.answer(state, questions, *media)except DecisionUnsupportedError as exc:raise Unavailable(501, "not_supported_error", str(exc)) from Noneexcept DecisionRequestError as exc:@@ -2144,16 +2191,23 @@ def decide(state,questions: dict[str, dict[str, Any]],images: list[str] | None = None,+ audio: list[str] | None = None,) -> dict[str, Any]:if not _admission.acquire(blocking = False):raise Unavailable(529, "overloaded", "System One is busy; retry shortly", retry_after = 1)try:- return _route(checkpoint, state, questions, images or None)+ return _route(checkpoint, state, questions, images or None, audio or None)finally:_admission.release()-def _route(checkpoint: Checkpoint, state, questions, images) -> dict[str, Any]:+def _route(+ checkpoint: Checkpoint,+ state,+ questions,+ images,+ audio = None,+) -> dict[str, Any]:global _fallback_reasonfrom utils.systemone_settings import get_backend@@ -2166,11 +2220,11 @@ def _route(checkpoint: Checkpoint, state, questions, images) -> dict[str, Any]:# A request, unlike a settings read, waits for device detection, so the first one already knows of MLX.hardware.get_device()state_images = _state_has_images(state)- target, reason = select(checkpoint, images, questions, state_images = state_images)+ target, reason = select(checkpoint, images, questions, state_images = state_images, audio = audio)if checkpoint.layout != "laya":_fallback_reason = reasontry:- return _decide(target, state, questions, images)+ return _decide(target, state, questions, images, audio)except NativeContextOverflow as exc:# Auto answers on PyTorch only what its longer window can hold; at equal windows it would refuse too.from .catalog import clef_unsupported_reason@@ -2212,6 +2266,7 @@ def _decide(state,questions: dict[str, dict[str, Any]],images: list[str] | None = None,+ audio: list[str] | None = None,) -> dict[str, Any]:global _last_used, _device_nameagent = _agent_for(checkpoint, LOAD_WAIT_S)@@ -2227,7 +2282,7 @@ def _decide(_last_used = time.monotonic()return resultif _is_mlx(checkpoint):- result = agent.decide(state, questions, images)+ result = agent.decide(state, questions, images, audio)_last_used = time.monotonic()return {"model": checkpoint.name,diff --git a/studio/backend/core/training/decision_trainer.py b/studio/backend/core/training/decision_trainer.pyindex 0ed6e88d02b..96f723cf788 100644--- a/studio/backend/core/training/decision_trainer.py+++ b/studio/backend/core/training/decision_trainer.py@@ -129,6 +129,27 @@ def _read_local_rows(paths: list[str], load_dataset, warn: Callable[[str], None]return [row for row in rows if isinstance(row, dict)]+def _hold_audio(rows: list[dict]) -> None:+ # The audio files an S3 download rewrote rows to name are deleted with it, before training reads them.+ for row in rows:+ audio = row.get("audio")+ if audio is None:+ continue+ clips = audio if isinstance(audio, list) else [audio]+ held = [_held_clip(clip) for clip in clips]+ row["audio"] = held if isinstance(audio, list) else held[0]+++def _held_clip(clip):+ # The rewrite leaves a plain path string or a {"path"} dict naming the download.+ if isinstance(clip, str) and os.path.isabs(clip) and os.path.isfile(clip):+ return {"bytes": Path(clip).read_bytes(), "path": clip}+ if isinstance(clip, dict) and clip.get("bytes") is None and clip.get("path"):+ if os.path.isfile(clip["path"]):+ clip["bytes"] = Path(clip["path"]).read_bytes()+ return clip++def _eval_sample(rows, seed: int):# Every row holds at least one decision, so EVAL_MAX rows can fill the eval set.if len(rows) <= EVAL_MAX:@@ -161,6 +182,7 @@ def _load_rows(raise _Stopped("Training cancelled") from Nonetry:rows = _read_local_rows(download.files, load_dataset, warn)+ _hold_audio(_sliced(rows, config))finally:download.cleanup()elif config.get("hf_dataset"):@@ -201,10 +223,14 @@ def _load_rows(_read_local_rows(config["local_eval_datasets"], load_dataset, warn), seed)+ return _sliced(rows, config), eval_rows+++def _sliced(rows: list, config: dict) -> list:start, end = config.get("dataset_slice_start"), config.get("dataset_slice_end")- if start is not None or end is not None:- rows = rows[start or 0 : (len(rows) if end is None else end + 1)]- return rows, eval_rows+ if start is None and end is None:+ return rows+ return rows[start or 0 : (len(rows) if end is None else end + 1)]def _is_mlx() -> bool:diff --git a/studio/backend/core/training/s3_dataset.py b/studio/backend/core/training/s3_dataset.pyindex 709fe23404c..247f084efca 100644--- a/studio/backend/core/training/s3_dataset.py+++ b/studio/backend/core/training/s3_dataset.py@@ -266,14 +266,21 @@ def _rewrite_row(row, resolve) -> None:if not isinstance(row, dict):returnfor column, value in row.items():- replacement = resolve(value)+ row[column] = _rewritten(value, resolve)+++def _rewritten(value, resolve):+ replacement = resolve(value)+ if replacement is not None:+ return replacement+ if isinstance(value, dict):+ # HF undecoded audio: {"path": ..., "bytes": ...}+ replacement = resolve(value.get("path"))if replacement is not None:- row[column] = replacement- elif isinstance(value, dict):- # HF undecoded audio: {"path": ..., "bytes": ...}- replacement = resolve(value.get("path"))- if replacement is not None:- value["path"] = replacement+ value["path"] = replacement+ elif isinstance(value, list):+ return [_rewritten(item, resolve) for item in value]+ return valuedef _rewrite_json_manifest(manifest_path: str, resolve) -> None:diff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.pyindex dcab5fc7cef..dd85adf29ad 100644--- a/studio/backend/core/training/training.py+++ b/studio/backend/core/training/training.py@@ -451,6 +451,8 @@ def _apply_model_cache_pin(config: dict[str, Any], warnings: list[str]) -> None:config["actual_model_repo_id"] = Noneconfig["model_snapshot_path"] = Noneconfig["model_revision"] = None+ # Nothing is pinned, so an offline start's pin has nothing left to validate.+ config["require_validated_model_snapshot"] = Falsereturnrequested_pin = config.get("model_snapshot_path")require_validated_snapshot = bool(config.get("require_validated_model_snapshot"))diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.pyindex 69a9723319e..e151712e61d 100644--- a/studio/backend/routes/settings.py+++ b/studio/backend/routes/settings.py@@ -1609,13 +1609,15 @@ def _decision_description(checkpoint, mlx: bool) -> str:def _clef_availability(checkpoint, reason: Optional[str]) -> dict:from core.systemone import laya_runtime- if getattr(checkpoint, "layout", "laya") == laya_runtime.GGUF:+ only_mlx = getattr(checkpoint, "backend", None) == laya_runtime.MLX+ if only_mlx or getattr(checkpoint, "layout", "laya") == laya_runtime.GGUF:+ only = {} if only_mlx else {"llama_cpp_only": True}# Selectable under a PyTorch runtime: the runtime row then says to switch it.try:laya_runtime.select(checkpoint, preference = "auto")except laya_runtime.Unavailable as exc:- return {"llama_cpp_only": True, "available": False, "unavailable_reason": exc.message}- return {"llama_cpp_only": True}+ return {**only, "available": False, "unavailable_reason": exc.message}+ return onlyif reason is None or getattr(checkpoint, "layout", "laya") != "clef":return {}# llama.cpp serves Clef without CUDA or ROCm, and so does the MLX engine on Apple Silicon.diff --git a/studio/backend/routes/systemone.py b/studio/backend/routes/systemone.pyindex d52e3d196e4..873676329f3 100644--- a/studio/backend/routes/systemone.py+++ b/studio/backend/routes/systemone.py@@ -40,6 +40,9 @@ MAX_QUESTION_CHARS = 20_000MAX_IMAGES = 4MAX_IMAGE_BYTES = 4 * 1024 * 1024MAX_IMAGES_BYTES = 8 * 1024 * 1024+# As many clips as unsloth-zoo's MLX engine takes; 40 MiB holds the 20 minutes it hears as 16 kHz mono WAV.+MAX_AUDIO = 8+MAX_AUDIO_BYTES = 40 * 1024 * 1024# What llama.cpp's mtmd decodes in-process (stb_image); WebP needs an ffmpeg it may not have._IMAGE_TYPES = {"image/png": "PNG", "image/jpeg": "JPEG"}_TYPES = ("noul", "choice", "score")@@ -69,6 +72,7 @@ class SystemOneRequest(BaseModel):questions: dict[str, QuestionIn] = Field(min_length = 1)# OpenJev's image extension: base64 data URLs, served by a Clef model through llama.cpp.images: Optional[list[str]] = None+ audio: Optional[list[str]] = Nonedef _error(@@ -161,7 +165,9 @@ async def system_one()if checkpoint is None:raise _error(400, "api_usage_error", f"Unknown model: {payload.model}")- result = await _decide(checkpoint, payload.state, payload.questions, payload.images)+ result = await _decide(+ checkpoint, payload.state, payload.questions, payload.images, payload.audio+ )headers = {"x-typesafe-request-id": str(uuid4())}if isinstance(checkpoint, catalog.Checkpoint):headers["x-unsloth-decision-backend"] = result.pop("_backend", "pytorch")@@ -213,6 +219,35 @@ def _validate_images(images: list[str]) -> None:)+def _validate_audio(audio: list[str]) -> None:+ # Shape and size only: the engine that hears the clips decodes them, and refuses what it cannot.+ if len(audio) > MAX_AUDIO:+ raise _error(422, "invalid_request_error", f"At most {MAX_AUDIO} audio clips per request")+ total = 0+ for index, url in enumerate(audio):+ header, comma, data = url.partition(",")+ kind = header.removeprefix("data:audio/").removesuffix(";base64")+ if not comma or not kind or header != f"data:audio/{kind};base64":+ raise _error(+ 422,+ "invalid_request_error",+ f"audio[{index}] must be a base64 audio data URL; remote URLs are not fetched",+ )+ total += 3 * len(data) // 4+ if not data or total > MAX_AUDIO_BYTES:+ raise _error(+ 422,+ "invalid_request_error",+ f"All audio clips together must be at most {MAX_AUDIO_BYTES // 1024**2} MiB",+ )+ try:+ base64.b64decode(data, validate = True)+ except (binascii.Error, ValueError):+ raise _error(+ 422, "invalid_request_error", f"audio[{index}] is not valid base64"+ ) from None++MAX_IMAGE_PIXELS = 4096 * 4096@@ -240,6 +275,7 @@ async def _decide(state: JSONContent,questions: dict[str, QuestionIn],images: list[str] | None = None,+ audio: list[str] | None = None,) -> dict:if not questions:raise _error(422, "invalid_request_error", "At least one question is required")@@ -260,6 +296,12 @@ async def _decide(raise _error(400, "api_usage_error", "Images are not forwarded to Decision API connections")+ if audio:+ _validate_audio(audio)+ if isinstance(checkpoint, catalog.Connection):+ raise _error(+ 400, "api_usage_error", "Audio is not forwarded to Decision API connections"+ )if isinstance(checkpoint, catalog.Connection):return await _connection_decide(@@ -274,6 +316,7 @@ async def _decide(state,{name: q.model_dump() for name, q in questions.items()},images or None,+ audio or None,)except laya_runtime.Unavailable as exc:raise _error(exc.status, exc.error_type, exc.message, exc.retry_after) from Nonediff --git a/studio/backend/tests/test_decision_serving.py b/studio/backend/tests/test_decision_serving.pyindex ce081f79660..9e8eae9995f 100644--- a/studio/backend/tests/test_decision_serving.py+++ b/studio/backend/tests/test_decision_serving.py@@ -914,7 +914,7 @@ def test_clef_serves_on_nvidia_and_amd_gpus(home, client, clef, monkeypatch, kinassert all(m["available"] and m["unavailable_reason"] is Nonefor m in models- if not m["llama_cpp_only"]+ if not m["llama_cpp_only"] and m["name"] != "clef-omni")@@ -1083,7 +1083,7 @@ def test_the_mlx_runtime_serves_only_through_the_engine(home, client, engine, moassert _put(client, device = "cpu").status_code == 200settings = client.get("/api/settings/systemone").json()assert not settings["mlx_available"] and settings["effective_backend"] is None- assert not any("MLX" in m["description"] for m in settings["models"])+ assert not any("or MLX" in m["description"] for m in settings["models"])assert "needs Apple Silicon" in settings["fallback_reason"] and _post(client).status_code == 400plan = client.get("/api/settings/systemone/resolve", params = {"backend": "mlx"}).json()assert (plan["repo"], plan["cached"]) == (None, False) and "needs Apple Silicon" in plan[@@ -1242,6 +1242,7 @@ def test_images_in_the_state_route_like_the_images_field(home, monkeypatch):questions = None,preference = None,state_images = False,+ audio = None,):seen.append(state_images and not images)raise laya_runtime.Unavailable(400, "api_usage_error", "stop")@@ -1336,6 +1337,105 @@ def test_a_clef_with_its_vision_tower_reads_images_through_the_mlx_engine(assert engine.read == ("s", ())+def test_a_clef_with_its_audio_tower_hears_through_the_mlx_engine(+ home, client, engine, monkeypatch+):+ served, folder = _clef_fine_tune(home, "clef_mlx_4"), home / "clef_mlx_4"+ tuned, kev = catalog.fine_tune(served), catalog.CHECKPOINTS["kev-4b"]+ zoo, clip = sys.modules["unsloth_zoo.mlx.decision"], "data:audio/wav;base64,AA=="+ image, questions = "data:image/png;base64,AA==", {"q": {"type": "noul", "instructions": "i"}}+ (folder / "processor_config.json").write_text("{}", encoding = "utf-8")++ def hears(checkpoint):+ try:+ routed = laya_runtime.select(checkpoint, audio = [clip])[0].backend == "mlx"+ except laya_runtime.Unavailable as refused:+ routed = "does not hear audio here" not in refused.message+ assert routed == ("audio" in laya_runtime.input_modalities(checkpoint))+ return routed++ # An audio tower in the folder and an unsloth-zoo that hears: neither alone, and no other entry.+ zoo.ClefModel = SimpleNamespace(takes_images = True, takes_audio = True)+ (folder / "config.json").write_text('{"vision_config": {}}', encoding = "utf-8")+ assert not hears(tuned)+ omni = {"thinker_config": {"audio_config": {}, "vision_config": {}}}+ (folder / "config.json").write_text(json.dumps(omni), encoding = "utf-8")+ assert hears(tuned) and not hears(kev) and not hears(catalog.CHECKPOINTS["clef-flash"])+ assert laya_runtime.input_modalities(tuned) == ["text", "image", "audio"]+ # Phi-4-multimodal and Nemotron Omni name their towers otherwise.+ phi4 = {"embd_layer": {"image_embd_layer": {}, "audio_embd_layer": {}}}+ for config, modalities in (+ (phi4, ["text", "image", "audio"]),+ ({"vision_config": {}, "sound_config": {}}, ["text", "image", "audio"]),+ ):+ (folder / "config.json").write_text(json.dumps(config), encoding = "utf-8")+ assert hears(tuned) and laya_runtime.input_modalities(tuned) == modalities+ (folder / "config.json").write_text(+ json.dumps({"embd_layer": {"audio_embd_layer": {}}}), encoding = "utf-8"+ )+ # A model that hears but does not see still reports the audio it is routed.+ assert hears(tuned) and laya_runtime.input_modalities(tuned) == ["text", "audio"]+ (folder / "config.json").write_text(json.dumps(omni), encoding = "utf-8")+ zoo.ClefModel = SimpleNamespace(takes_images = True)+ assert not hears(tuned)+ zoo.ClefModel = SimpleNamespace(takes_images = True, takes_audio = True)+ for runtime in ("pytorch", "llama.cpp"):+ with pytest.raises(laya_runtime.Unavailable, match = "Audio is served only by MLX"):+ laya_runtime.select(tuned, audio = [clip], preference = runtime)+ assert _put(client, enabled = True, model = served).status_code == 200+ clips = [clip, "data:audio/wav;base64,AQ=="]+ body = {"model": "default", "state": "s", "questions": questions, "audio": clips}+ assert client.post("/v1/systemone", json = body).headers["x-unsloth-decision-backend"] == "mlx"+ assert engine.read == ("s", (None, clips))+ assert client.post("/v1/systemone", json = {**body, "model": "kev-4b"}).status_code == 400+ assert laya_runtime._route(tuned, "s", questions, [image], clips)["_backend"] == "mlx"+ assert engine.read == ("s", ([image], clips))+ monkeypatch.setattr(systemone, "MAX_AUDIO_BYTES", 6)+ for audio, message in (+ ([clip] * 9, "At most 8 audio clips"),+ ([clip, image], "audio[1] must be a base64 audio data URL"),+ ([clip, "data:audio/wav;base64,@@@@"], "audio[1] is not valid base64"),+ ([clip] * 3, "All audio clips together"),+ ):+ refused = client.post("/v1/systemone", json = {**body, "audio": audio})+ assert refused.status_code == 422 and message in refused.json()["detail"]["message"]+ asked, connection = {"q": systemone.QuestionIn(type = "noul")}, catalog.Connection("p", "m")+ with pytest.raises(systemone.HTTPException, match = "Audio is not forwarded"):+ asyncio.run(systemone._decide(connection, "s", asked, None, [clip]))+++def test_clef_omni_is_served_by_the_mlx_engine_or_not_at_all(home, client, engine, monkeypatch):+ omni, clip = catalog.CHECKPOINTS["clef-omni"], "data:audio/wav;base64,AA=="+ zoo = sys.modules["unsloth_zoo.mlx.decision"]+ zoo.ClefModel = SimpleNamespace(takes_images = True, takes_audio = True)+ assert _put(client, enabled = True, model = "clef-omni").status_code == 200+ settings = client.get("/api/settings/systemone").json()+ assert settings["effective_backend"] == "mlx"+ assert settings["input_modalities"] == ["text", "image", "audio"]+ questions = {"q": {"type": "noul", "instructions": "i"}}+ body = {"model": "default", "state": "s", "questions": questions, "audio": [clip]}+ assert client.post("/v1/systemone", json = body).headers["x-unsloth-decision-backend"] == "mlx"+ assert engine.loaded == [("clef-omni-8bit", "clef", None)] and engine.read[1] == (None, [clip])+ assert not laya_runtime.input_modalities(catalog.CHECKPOINTS["clef"])[2:]+ # It needs no PyTorch, so a --no-torch install on Apple Silicon enables and lists it.+ with monkeypatch.context() as patch:+ patch.setattr(systemone_settings, "runtime_unavailable_reason", lambda: "no torch")+ systemone_settings.validate(enabled = True, model = "clef-omni")+ with pytest.raises(ValueError, match = "no torch"):+ systemone_settings.validate(enabled = True, model = "clef")+ listed = [m["id"] for m in systemone.decision_model_objects()]+ assert "clef-omni" in listed and "clef" not in listed+ for runtime in ("pytorch", "llama.cpp"):+ with pytest.raises(laya_runtime.Unavailable, match = "clef-omni is served only by MLX"):+ laya_runtime.select(omni, preference = runtime)+ # Where the engine is not, the entry is listed as one this machine cannot serve.+ _spoof_device(monkeypatch, "cuda")+ listed = {m["name"]: m for m in client.get("/api/settings/systemone").json()["models"]}+ assert listed["clef-omni"]["available"] is False and not listed["clef-omni"]["llama_cpp_only"]+ assert "served only by MLX" in listed["clef-omni"]["unavailable_reason"]+ assert _post(client, "clef-omni").status_code == 400++def test_auto_keeps_images_in_the_state_off_mlx(home, engine, monkeypatch):kev = catalog.CHECKPOINTS["kev-4b"]monkeypatch.setattr(laya_runtime, "_native_unavailable", lambda *args: None)diff --git a/studio/backend/tests/test_decision_training.py b/studio/backend/tests/test_decision_training.pyindex f94afd6c0a9..d2172ea48d9 100644--- a/studio/backend/tests/test_decision_training.py+++ b/studio/backend/tests/test_decision_training.py@@ -758,6 +758,51 @@ def test_a_stop_during_the_s3_download_cancels_the_run(monkeypatch):_load(lambda: True, s3_config = {"bucket": "b"})+def test_s3_audio_rows_keep_their_clips_after_the_download_is_cleaned_up(monkeypatch, tmp_path):+ import shutil+ from types import SimpleNamespace++ from core.training import s3_dataset++ clips = [tmp_path / "a.wav", tmp_path / "b.wav", tmp_path / "c.wav"]+ for index, clip in enumerate(clips):+ clip.write_bytes(b"RIFF" + bytes([index]))+ manifest = tmp_path / "rows.jsonl"+ rows = [+ {**_row(0), "audio": {"bytes": None, "path": str(clips[0])}},+ {**_row(1), "audio": [{"bytes": None, "path": str(clips[1])}]},+ # A plain path string, as the S3 rewrite leaves a manifest's usual audio cell.+ {**_row(2), "audio": str(clips[2])},+ _row(3),+ ]+ manifest.write_text("\n".join(json.dumps(row) for row in rows), encoding = "utf-8")+ copy = tmp_path / "download"+ copy.mkdir()+ shutil.copy(manifest, copy)++ def download(s3_config, cancel_callback):+ return SimpleNamespace(+ files = [str(copy / "rows.jsonl")], cleanup = lambda: [clip.unlink() for clip in clips]+ )++ monkeypatch.setattr(s3_dataset, "prepare_s3_dataset_download", download)+ loaded, _, _ = _load(s3_config = {"bucket": "b"})+ assert [+ loaded[0]["audio"]["bytes"],+ loaded[1]["audio"][0]["bytes"],+ loaded[2]["audio"]["bytes"],+ ] == [b"RIFF\x00", b"RIFF\x01", b"RIFF\x02"]+ assert "audio" not in loaded[3]+ assert not any(clip.exists() for clip in clips)+ # A slice reads only the clips of the rows it keeps.+ for index, clip in enumerate(clips):+ clip.write_bytes(b"RIFF" + bytes([index]))+ read, read_bytes = [], Path.read_bytes+ monkeypatch.setattr(Path, "read_bytes", lambda path: read.append(path.name) or read_bytes(path))+ loaded, _, _ = _load(s3_config = {"bucket": "b"}, dataset_slice_start = 1, dataset_slice_end = 1)+ assert [row["audio"][0]["bytes"] for row in loaded] == [b"RIFF\x01"] and read == ["b.wav"]++def test_other_accounts_reach_only_the_configured_fine_tune(studio_home, client):from utils.account_context import AccountContext, bind_account, reset_accountfrom utils.paths import outputs_rootdiff --git a/studio/backend/tests/test_decision_training_route.py b/studio/backend/tests/test_decision_training_route.pyindex b7bd8d323fe..79d80114d65 100644--- a/studio/backend/tests/test_decision_training_route.py+++ b/studio/backend/tests/test_decision_training_route.py@@ -497,6 +497,30 @@ def test_a_hub_checkpoint_downloads_under_the_stall_watchdog(monkeypatch, tmp_paassert events == [] and len(downloads) == 1+def test_an_offline_decision_start_does_not_wait_for_a_pin_it_never_uses(tmp_path):+ from core.training.training import _apply_cache_pins+ from core.training.worker import _verify_config_pins++ events = []+ queue = SimpleNamespace(put = events.append)+ # Offline, the start route pins the cached snapshot of an LLM given a decision head, as of any LLM.+ snapshot = str(tmp_path / "snapshot")+ pinned = {+ "model_name": "unsloth/Qwen3.5-0.8B",+ "actual_model_repo_id": "unsloth/Qwen3.5-0.8B",+ "model_snapshot_path": snapshot,+ "require_validated_model_snapshot": True,+ }+ decision = {**pinned, "is_decision": True, "decision_layout": "llm"}+ _apply_cache_pins(decision)+ assert _verify_config_pins(decision, queue) and events == []+ # An LLM run still loads exactly the snapshot it was pinned to, or not at all.+ from core.training.provenance import ExactResumeResourcesUnavailable++ with pytest.raises(ExactResumeResourcesUnavailable, match = "no longer available"):+ _apply_cache_pins(pinned)++@pytest.mark.skipif(sys.platform == "darwin", reason = "Apple Silicon trains with MLX")def test_decision_worker_uses_one_gpu_and_no_llm_kernel_installs(monkeypatch, tmp_path):import multiprocessing as mpdiff --git a/studio/backend/tests/test_s3_audio_dataset.py b/studio/backend/tests/test_s3_audio_dataset.pyindex e3488538adf..39cc07eb477 100644--- a/studio/backend/tests/test_s3_audio_dataset.py+++ b/studio/backend/tests/test_s3_audio_dataset.py@@ -148,6 +148,7 @@ def test_jsonl_audio_references_are_rewritten_to_local_paths(monkeypatch, tmp_pajson.dumps({"audio": {"path": "audio/a.wav"}, "text": "three"}),# Unmatched references are somebody else's contract: left alone.json.dumps({"audio": "https://example.com/c.wav", "text": "four"}),+ json.dumps({"audio": [{"path": "audio/a.wav"}, "audio/b.mp3"], "text": "five"}),])_install(@@ -165,6 +166,7 @@ def test_jsonl_audio_references_are_rewritten_to_local_paths(monkeypatch, tmp_paassert rows[1]["audio"] == local_bassert rows[2]["audio"]["path"] == local_aassert rows[3]["audio"] == "https://example.com/c.wav"+ assert rows[4]["audio"] == [{"path": local_a}, local_b]assert all(os.path.isabs(p) for p in (rows[0]["audio"], rows[1]["audio"]))assert rows[0]["text"] == "one"diff --git a/studio/backend/utils/systemone_settings.py b/studio/backend/utils/systemone_settings.pyindex 6a1632a5fe8..77e3f9428c1 100644--- a/studio/backend/utils/systemone_settings.py+++ b/studio/backend/utils/systemone_settings.py@@ -71,9 +71,11 @@ def runtime_unavailable_reason() -> str | None:def llama_cpp_only(name: Any) -> bool:- """A GGUF-only catalog entry: llama.cpp serves it without PyTorch."""+ """A catalog entry served without PyTorch: a GGUF by llama.cpp, an MLX-only entry by unsloth-zoo's MLX engine."""from core.systemone.catalog import CHECKPOINTS- return getattr(CHECKPOINTS.get(name), "layout", None) == "gguf"++ entry = CHECKPOINTS.get(name)+ return getattr(entry, "layout", None) == "gguf" or getattr(entry, "backend", None) == "mlx"def get_enabled() -> bool:diff --git a/studio/frontend/src/config/training.ts b/studio/frontend/src/config/training.tsindex f86d415a46e..003ff34f84f 100644--- a/studio/frontend/src/config/training.ts+++ b/studio/frontend/src/config/training.ts@@ -232,10 +232,29 @@ export const PRIORITY_TRAINING_MODELS: readonly string[] = ["unsloth/Llama-3.2-3B-Instruct",];+const LAYA_TRAINING_MODEL = "convaiinnovations/laya";+const CLEF_OMNI_MLX_TRAINING_MODEL = "mlx-community/clef-omni-8bit";++/** The pinned models for a device: Clef Omni trains only through MLX, so only a Mac lists it. */+export function priorityTrainingModels(+ deviceType?: string | null,+): readonly string[] {+ return deviceType === "mac"+ ? PRIORITY_TRAINING_MODELS.flatMap((id) =>+ id === LAYA_TRAINING_MODEL ? [id, CLEF_OMNI_MLX_TRAINING_MODEL] : [id],+ )+ : PRIORITY_TRAINING_MODELS;+}+/** Pin priority models to the top of a list of model IDs, preserving their defined order. */-export function applyPriorityOrdering(ids: string[]): string[] {+export function applyPriorityOrdering(+ ids: string[],+ deviceType?: string | null,+): string[] {const idSet = new Set(ids);- const pinned = PRIORITY_TRAINING_MODELS.filter((id) => idSet.has(id));+ const pinned = priorityTrainingModels(deviceType).filter((id) =>+ idSet.has(id),+ );const pinnedSet = new Set(pinned);const rest = ids.filter((id) => !pinnedSet.has(id));return [...pinned, ...rest];diff --git a/studio/frontend/src/features/settings/lib/decision-model-labels.ts b/studio/frontend/src/features/settings/lib/decision-model-labels.tsindex b2a3c37a7c5..9c855a0f42f 100644--- a/studio/frontend/src/features/settings/lib/decision-model-labels.ts+++ b/studio/frontend/src/features/settings/lib/decision-model-labels.ts@@ -14,6 +14,9 @@ export const DECISION_MODEL_LABELS: Record<string, TranslationKey> = {/** Cloudflare Clef models are named by themselves; the Laya labels read as "Laya <label>". */export function isClefDecisionModel(name: string | null | undefined): boolean {return (- name === "clef" || name === "clef-flash" || !!name?.startsWith("clef-ft:")+ name === "clef" ||+ name === "clef-flash" ||+ name === "clef-omni" ||+ !!name?.startsWith("clef-ft:"));}diff --git a/studio/frontend/src/features/train-model-picker/components/train-model-selector.tsx b/studio/frontend/src/features/train-model-picker/components/train-model-selector.tsxindex 2ce2bb1d3e1..4882e58f898 100644--- a/studio/frontend/src/features/train-model-picker/components/train-model-selector.tsx+++ b/studio/frontend/src/features/train-model-picker/components/train-model-selector.tsx@@ -18,8 +18,8 @@ import { usePickerState } from "@/components/resource-picker/use-picker-state";import { usePlatformStore } from "@/config/env";import {MODEL_TYPE_TO_HF_TASKS,- PRIORITY_TRAINING_MODELS,applyPriorityOrdering,+ priorityTrainingModels,} from "@/config/training";import {type CachedInventoryRow,@@ -509,7 +509,7 @@ export function TrainModelSelector({task: trainingModelTaskFilter(requiredModelType),accessToken: hfApiToken(picker.debouncedHfToken),excludeGguf: true,- priorityIds: PRIORITY_TRAINING_MODELS,+ priorityIds: priorityTrainingModels(deviceType),ownerScope: picker.debouncedHubQuery.trim() ? "all" : "unsloth",enabled: online && picker.open && tab === PICKER_TAB.hub,});@@ -534,8 +534,8 @@ export function TrainModelSelector({) {ids.push(selectedModel);}- return applyPriorityOrdering(ids);- }, [hfResults, modelKnownCached, modelLocalPath, selectedModel]);+ return applyPriorityOrdering(ids, deviceType);+ }, [deviceType, hfResults, modelKnownCached, modelLocalPath, selectedModel]);const hfResultById = useMemo(() => {const map = new Map<string, HfModelResult>();diff --git a/studio/frontend/src/features/training/api/mappers.ts b/studio/frontend/src/features/training/api/mappers.tsindex 3e4318dd732..3dc40a6d2e3 100644--- a/studio/frontend/src/features/training/api/mappers.ts+++ b/studio/frontend/src/features/training/api/mappers.ts@@ -304,7 +304,8 @@ export function buildTrainingStartPayload(finetune_attention_modules: config.finetuneAttentionModules,finetune_mlp_modules: config.finetuneMLPModules,is_dataset_image: isEmbedding ? false : !!config.isDatasetImage,- is_dataset_audio: isEmbedding ? false : config.isDatasetAudio,+ // The decision trainer reads its rows itself; this flag selects the chat audio path.+ is_dataset_audio: isEmbedding || isDecision ? false : config.isDatasetAudio,is_embedding: isEmbedding && !isDecision,is_decision: isDecision,model_subfolder: isDecision ? config.modelSubfolder : null,diff --git a/studio/frontend/src/features/training/hooks/use-training-readiness.ts b/studio/frontend/src/features/training/hooks/use-training-readiness.tsindex cd0d1153064..e775342dc13 100644--- a/studio/frontend/src/features/training/hooks/use-training-readiness.ts+++ b/studio/frontend/src/features/training/hooks/use-training-readiness.ts@@ -45,7 +45,8 @@ export function deriveTrainingReadiness(state.modelType === "decision" && modelError !== null;const isModelCapabilitiesSettled = hasModel && !isLoadingModel;const isIncompatible =- isModelCapabilitiesSettled && hasIncompatibleTrainingModalities(state);+ isModelCapabilitiesSettled &&+ hasIncompatibleTrainingModalities(state, deviceType);const hasDatasetModalityMetadata = state.isDatasetImage !== null;const modelHandlesAllModalities = state.isVisionModel && state.isAudioModel;const datasetUnverified =diff --git a/studio/frontend/src/features/training/lib/model-type-inference.ts b/studio/frontend/src/features/training/lib/model-type-inference.tsindex b289d430b07..1ac1156ab0f 100644--- a/studio/frontend/src/features/training/lib/model-type-inference.ts+++ b/studio/frontend/src/features/training/lib/model-type-inference.ts@@ -9,7 +9,7 @@ import {import type { ModelTypeCapabilityFlags } from "./model-type-capabilities";const TEXT_MODEL_TAGS = new Set(["text-generation"]);-const DECISION_MODEL_TAGS = new Set(["laya"]);+const DECISION_MODEL_TAGS = new Set(["laya", "clef"]);export type TrainingModelTypeMetadata = TrainingModelModalityMetadata;diff --git a/studio/frontend/src/features/training/lib/validation.ts b/studio/frontend/src/features/training/lib/validation.tsindex 0856ac9c6d3..7ee328e946d 100644--- a/studio/frontend/src/features/training/lib/validation.ts+++ b/studio/frontend/src/features/training/lib/validation.ts@@ -44,11 +44,24 @@ function isTrainingConfigSupportedOnDevice(if (deviceType !== "mac") {return true;}- return !(config.isEmbeddingModel || config.isDatasetAudio === true);+ return !(config.isEmbeddingModel || hasChatAudioRows(config, deviceType));+}++/** A Mac Clef (or Clef-head LLM) trainer reads its own audio rows; elsewhere, and for Laya, they train as text. */+function hasChatAudioRows(+ config: TrainingConfigState,+ deviceType?: string,+): boolean {+ const trainerReadsRows =+ deviceType === "mac" &&+ config.modelType === "decision" &&+ (config.decisionLayout === "clef" || config.decisionLayout === "llm");+ return config.isDatasetAudio === true && !trainerReadsRows;}export function hasIncompatibleTrainingModalities(config: TrainingConfigState,+ deviceType?: string,): boolean {return ((!config.isVisionModel && config.isDatasetImage === true) ||@@ -58,7 +71,7 @@ export function hasIncompatibleTrainingModalities(// real download error, which is what a gated text model already does.(!config.audioCapabilityUnknown &&!config.isAudioModel &&- config.isDatasetAudio === true)+ hasChatAudioRows(config, deviceType)));}diff --git a/studio/frontend/tests/training-decision-model.test.ts b/studio/frontend/tests/training-decision-model.test.tsindex 1d456f09807..27c2c10183e 100644--- a/studio/frontend/tests/training-decision-model.test.ts+++ b/studio/frontend/tests/training-decision-model.test.ts@@ -14,8 +14,13 @@ registerStoreStubResolver();installLocalStorageFake();const yaml = await import("js-yaml");-const { DEFAULT_HYPERPARAMS, LR_DEFAULT_DECISION_FULL, LR_DEFAULT_LORA } =- await import("../src/config/training.ts");+const {+ DEFAULT_HYPERPARAMS,+ LR_DEFAULT_DECISION_FULL,+ LR_DEFAULT_LORA,+ applyPriorityOrdering,+ priorityTrainingModels,+} = await import("../src/config/training.ts");const { setAuthFetchHandler } = await import("./helpers/store-stubs/auth.ts");const { useTrainingConfigStore } = await import("../src/features/training/stores/training-config-store.ts"@@ -29,6 +34,9 @@ const { DatasetFormatError } = await import(const { validateTrainingConfig } = await import("../src/features/training/lib/validation.ts");+const { classifyUnslothSupport } = await import(+ "../src/features/hub/lib/unsloth-support.ts"+);const { deriveTrainingReadiness } = await import("../src/features/training/hooks/use-training-readiness.ts");@@ -527,6 +535,65 @@ test("the model picker knows Laya is a decision model before its config loads",assert.equal(isTrainingModelTypeSupportedOnDevice(modelType, "cuda"), true);});+test("a Mac lists Clef Omni and trains it on rows that carry audio", () => {+ const omni = "mlx-community/clef-omni-8bit";+ const metadata = {+ pipelineTag: "zero-shot-classification",+ tags: ["mlx", "safetensors", "qwen3_omni_moe", "clef", "audio", "video"],+ };+ assert.equal(+ inferTrainingModelTypeFromFlags(+ trainingModelTypeFlagsFromMetadata({+ pipelineTag: "image-text-to-text",+ tags: ["clef", "text-to-audio"],+ }),+ ),+ "decision",+ );+ assert.equal(+ inferTrainingModelTypeFromFlags(trainingModelTypeFlagsFromMetadata(metadata)),+ "decision",+ );+ const support = classifyUnslothSupport({+ modelId: omni,+ libraryName: "mlx",+ deviceType: "mac",+ ...metadata,+ });+ assert.equal(support.status, "supported");+ assert.equal(priorityTrainingModels("cuda").includes(omni), false);+ assert.deepEqual(applyPriorityOrdering(["unsloth/x", omni, LAYA], "mac"), [+ LAYA,+ omni,+ "unsloth/x",+ ]);++ const config = {+ ...useTrainingConfigStore.getState(),+ ...HF_DECISION_DATASET,+ selectedModel: omni,+ modelType: "decision" as const,+ decisionLayout: "clef" as const,+ isDatasetAudio: true,+ };+ assert.equal(deriveTrainingReadiness(config, "mac", true).isReady, true);+ // So can an LLM that is given a decision head, such as a plain omni model.+ const fresh = { ...config, decisionLayout: "llm" as const };+ assert.equal(deriveTrainingReadiness(fresh, "mac", true).isReady, true);+ // The backend refuses the chat audio flag on a Mac, and the decision trainer never reads it.+ assert.equal(buildTrainingStartPayload(config, null).is_dataset_audio, false);+ // Only the MLX Clef trainer reads the clips: Laya, another device and a chat model would not.+ for (const [other, device] of [+ [{ decisionLayout: "laya" as const }, "mac"],+ [{}, "cuda"],+ [{ modelType: "text" as const }, "mac"],+ ] as const) {+ const blocked = deriveTrainingReadiness({ ...config, ...other }, device, true);+ assert.equal(blocked.isReady, false);+ assert.equal(blocked.isIncompatible, true);+ }+});+test("only the Studio owner can start a decision run", () => {const config = {...useTrainingConfigStore.getState(),diff --git a/tests/test_decision_model_mlx.py b/tests/test_decision_model_mlx.pyindex 48ff1de000c..40d4b671127 100644--- a/tests/test_decision_model_mlx.py+++ b/tests/test_decision_model_mlx.py@@ -435,12 +435,18 @@ def test_the_images_of_a_row_reach_the_clef_trainer_and_predict(seen, real, keep = [], zoo.clef_training_item, [True]- def item(pipeline, state, questions, max_length, *images):- # A row without images is asked as before, with no fifth argument.- seen.append([getattr(image, "size", image) for image in images[0]] if images else None)+ def item(+ pipeline,+ state,+ questions,+ max_length,+ images = None,+ audio = None,+ ):+ seen.append([getattr(image, "size", image) for image in images] if images else audio)return {**real(pipeline, state, questions, max_length),- **({"images": images[0]} if keep and images else {}),+ **({"images": images} if keep and images else {}),}monkeypatch.setattr(zoo, "clef_training_item", item)@@ -465,6 +471,40 @@ def test_the_images_of_a_row_reach_the_clef_trainer_and_predict(model, tokenizer, "s", {"q": {"type": "noul", "instructions": "i"}}, images = url)assert seen[-1] == [url]+ # Audio rows: data URLs as they are, an undecoded column's bytes or file as one, a decoded clip with its rate.+ (tmp_path / "a.wav").write_bytes(b"RIFF")+ sound = "data:audio/wav;base64," + base64.b64encode(b"RIFF").decode()+ clip = {"path": None, "array": [0.0, 1.0], "sampling_rate": 16000}++ class Decoder:+ # What datasets 4 decodes an audio column to.+ get_all_samples = None++ def __getitem__(self, key):+ return clip[key]++ class Corrupt(Decoder):+ def __getitem__(self, key):+ raise RuntimeError("invalid data found")++ heard = [+ {**_row(0), "audio": [sound, {"bytes": b"RIFF"}]},+ {**_row(1), "audio": {"bytes": None, "path": str(tmp_path / "a.wav")}},+ {**_row(2), "audio": [None, {"bytes": None, "path": None}, clip, Decoder()]},+ {**_row(3), "audio": {"bytes": None, "path": str(tmp_path / "none.wav")}},+ ]+ items, report = FastDecisionModel.build_dataset(+ [_row(0), {**_row(1), "audio": [Decoder(), Corrupt()]}], tokenizer, model+ )+ assert [item["row"] for item in items] == [0] and "could not be decoded" in report["reason"]+ del seen[:]+ items, report = FastDecisionModel.build_dataset(heard, tokenizer, model)+ assert [item["row"] for item in items] == [0, 1, 2] and "none.wav" in report["reason"]+ assert seen == [[sound, sound], [sound], [{"array": [0.0, 1.0], "sampling_rate": 16000}] * 2]+ FastDecisionModel.predict(+ model, tokenizer, "s", {"q": {"type": "noul", "instructions": "i"}}, audio = sound+ )+ assert seen[-1] == [sound]# An unsloth-zoo that takes no images says so, where skipping the rows would train without them.monkeypatch.setattr(zoo,@@ -477,7 +517,9 @@ def test_the_images_of_a_row_reach_the_clef_trainer_and_predict(question = {"q": {"type": "noul", "instructions": "i"}}for call in (lambda: FastDecisionModel.build_dataset(rows, tokenizer, model),+ lambda: FastDecisionModel.build_dataset(heard, tokenizer, model),lambda: FastDecisionModel.predict(model, tokenizer, "s", question, images = url),+ lambda: FastDecisionModel.predict(model, tokenizer, "s", question, audio = sound),):with pytest.raises(ImportError, match = "newer unsloth-zoo"):call()@@ -486,6 +528,8 @@ def test_the_images_of_a_row_reach_the_clef_trainer_and_predict(FastDecisionModel.predict(laya, laya_tokenizer, "s", {"q": {"type": "noul", "instructions": "i"}}, images = [url])+ with pytest.raises(unsloth._decision_mlx.DecisionDataError, match = "Laya reads text only"):+ FastDecisionModel.predict(laya, laya_tokenizer, "s", question, audio = sound)assert FastDecisionModel.predict(laya, laya_tokenizer, "s", {"q": {"type": "noul", "instructions": "i"}}, images = "[]")diff --git a/unsloth/models/decision_mlx.py b/unsloth/models/decision_mlx.pyindex 69cd854ae21..ea77caf6ebc 100644--- a/unsloth/models/decision_mlx.py+++ b/unsloth/models/decision_mlx.py@@ -58,9 +58,37 @@ from unsloth._decision_common import ()+def _row_audio(row: dict) -> list:+ # Data URLs, decoded {"array", "sampling_rate"} 16 kHz clips, or an undecoded column's {"bytes", "path"}.+ audio = _parsed(row.get("audio"))+ if audio is None:+ return []+ found = []+ for clip in audio if isinstance(audio, (list, tuple)) else [audio]:+ if hasattr(clip, "get_all_samples"):+ # datasets 4 decodes an audio column to a decoder that reads like the dict; a clip it cannot decode fails its row.+ try:+ clip = {"array": clip["array"], "sampling_rate": clip["sampling_rate"]}+ except RuntimeError as exc:+ raise ValueError(f"an audio clip could not be decoded: {exc}") from None+ elif isinstance(clip, dict) and "array" in clip:+ clip = {"array": clip["array"], "sampling_rate": clip.get("sampling_rate")}+ elif isinstance(clip, dict) and set(clip) <= {"bytes", "path"}:+ import base64++ data = clip.get("bytes")+ if data is None and clip.get("path"):+ data = Path(clip["path"]).read_bytes()+ # The container is recognized from the bytes, whatever the URL calls it.+ clip = data and "data:audio/wav;base64," + base64.b64encode(data).decode()+ if clip:+ found.append(clip)+ return found++def _decision_clef_items(rows, options: Callable, encode: Callable, validate, report, skip) -> list:# One item per row. `options(question)` names a question's options in the head's order and- # `encode(state, questions, images)` returns the backend's tokenized fields, raising when they do not fit.+ # `encode(state, questions, images, audio)` returns the backend's tokenized fields, raising when they do not fit.items = []for index, row in enumerate(rows):row = row if isinstance(row, dict) else {}@@ -94,7 +122,7 @@ def _decision_clef_items(rows, options: Callable, encode: Callable, validate, reif not kept:continuetry:- encoded = encode(state, kept, _row_images(row))+ encoded = encode(state, kept, _row_images(row), _row_audio(row))except (TypeError, ValueError, OSError) as exc:for name in kept:skip(index, f"does not fit: {exc}", name)@@ -415,16 +443,25 @@ def _decision_pad_token_id(tokenizer) -> int:return getattr(tokenizer, "tokenizer", tokenizer).pad_token_id-def _clef_item(zoo, pipeline, state, questions, max_length, images):- if not images:- return zoo.clef_training_item(pipeline, state, questions, max_length)- if "images" not in inspect.signature(zoo.clef_training_item).parameters:- # Left to fail as a row that does not fit, a dataset would train without its image rows.- raise ImportError(- "Unsloth: images in decision rows need a newer unsloth-zoo. "- "Upgrade with `pip install -U unsloth-zoo`."- )- return zoo.clef_training_item(pipeline, state, questions, max_length, images)+def _clef_item(+ zoo,+ pipeline,+ state,+ questions,+ max_length,+ images,+ audio = (),+):+ media = {name: value for name, value in (("images", images), ("audio", audio)) if value}+ taken = inspect.signature(zoo.clef_training_item).parameters+ for name in media:+ if name not in taken:+ # Left to fail as a row that does not fit, a dataset would train without these rows.+ raise ImportError(+ f"Unsloth: decision rows with {name} need a newer unsloth-zoo. "+ "Upgrade with `pip install -U unsloth-zoo`."+ )+ return zoo.clef_training_item(pipeline, state, questions, max_length, **media)def _clef_items(pipeline, rows, tokenizer, max_len, validate, report, skip) -> list:@@ -432,8 +469,8 @@ def _clef_items(pipeline, rows, tokenizer, max_len, validate, report, skip) -> lreturn _decision_clef_items(rows,functools.partial(zoo.clef_option_keys, pipeline),- lambda state, questions, images: _clef_item(- zoo, pipeline, state, questions, max_len, images+ lambda state, questions, images, audio: _clef_item(+ zoo, pipeline, state, questions, max_len, images, audio),validate,report,@@ -843,9 +880,11 @@ class FastDecisionModel:state,questions: dict,images = None,+ audio = None,) -> dict:"""Answers one Decision API request: {name: answer} at the calibrated temperatures.`images` (as in a row: PIL images, data URLs, or bytes / path dicts) are read by a Clef whose base model has a vision tower.+ `audio` (as in a row: data URLs, bytes / path dicts, or decoded 16 kHz mono clips) is heard by a Clef whose base model has an audio tower.Each answer is the Decision API's (choice / confidence, score / legend, or noul) plus"answer" (the option, True / False for noul, the level number for score) and"probabilities" over every option."""@@ -856,19 +895,19 @@ class FastDecisionModel:if not isinstance(questions, dict) or not questions:raise DecisionDataError("questions must be a non-empty dict of name to question")state, config, zoo = _parsed(state), model.decision_config, _decision_zoo()- images = _row_images({"images": images})+ images, audio = _row_images({"images": images}), _row_audio({"audio": audio})if _is_clef(model):pipeline = model._unsloth_pipelinequestions = {str(name): _clef_question(q) for name, q in questions.items()}# Read up to CLEF_SERVE_MAX_LEN tokens, like serving, even past the training cut.max_length = max(int(config.get("max_len", CLEF_MAX_LEN)), CLEF_SERVE_MAX_LEN)- item = _clef_item(zoo, pipeline, state, questions, max_length, images)+ item = _clef_item(zoo, pipeline, state, questions, max_length, images, audio)logits = zoo.clef_logits(model, [item])[0]keys = [zoo.clef_option_keys(pipeline, q) for q in questions.values()]kinds = [QUESTION_TYPES.index(q["type"]) for q in questions.values()]else:- if images:- raise DecisionDataError("Laya reads text only; images need a Clef model.")+ if images or audio:+ raise DecisionDataError("Laya reads text only; images and audio need a Clef model.")common, max_len = _laya().common, int(config.get("max_len", 512))tokenizer = getattr(tokenizer, "tokenizer", tokenizer)items, keys = [], []