Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

MLX: Clef Omni in Unsloth Studio, with audio in the Decision API and decision training (#13538)

Long Yixingdaab3bcd80cb
diff --git a/studio/backend/core/systemone/catalog.py b/studio/backend/core/systemone/catalog.py
index e8305d0db5f..4f4d9598385 100644
--- a/studio/backend/core/systemone/catalog.py
+++ b/studio/backend/core/systemone/catalog.py
@@ -31,6 +31,8 @@ class Checkpoint:
revision: str | None = None
# Display name where the frontend has none of its own.
label: str | None = None
+ # A Hub entry whose model hears audio; a local folder says so in its config.
+ audio: bool = False
@property
def is_local(self) -> bool:
@@ -83,6 +85,18 @@ CHECKPOINTS = {
54_976_000_000,
"clef",
),
+ Checkpoint(
+ "clef-omni",
+ "mlx-community/clef-omni-8bit",
+ None,
+ "Cloudflare Clef Omni on Qwen3-Omni 30B (8-bit, MLX only): text, images and audio, "
+ "needs Apple Silicon with 48 GB of memory or more.",
+ 35_044_686_208,
+ "clef",
+ "mlx",
+ label = "Clef Omni",
+ audio = True,
+ ),
_gguf(
"kev-0.8b",
"Kev 0.8B",
diff --git a/studio/backend/core/systemone/laya_runtime.py b/studio/backend/core/systemone/laya_runtime.py
index a421f7b0343..50e761a427f 100644
--- a/studio/backend/core/systemone/laya_runtime.py
+++ b/studio/backend/core/systemone/laya_runtime.py
@@ -160,8 +160,11 @@ def _engine_available() -> bool:
def _mlx_target(checkpoint: Checkpoint) -> Checkpoint | None:
"""What unsloth-zoo's MLX engine serves in process on Apple Silicon: a merged Clef's own folder, or the source repo of a GGUF entry."""
- if _is_mlx(checkpoint) or not _engine_available():
+ if not _engine_available():
return None
+ if _is_mlx(checkpoint):
+ # Only a catalog entry is its own MLX form; a target has none further.
+ return checkpoint if CHECKPOINTS.get(checkpoint.name) == checkpoint else None
if checkpoint.layout == "clef":
if checkpoint.is_local:
from utils.models.model_config import clef_folder_kind
@@ -180,18 +183,18 @@ def _mlx_target(checkpoint: Checkpoint) -> Checkpoint | None:
)
-def _mlx_reads_images(checkpoint: Checkpoint) -> bool:
- """Whether the MLX form of an entry reads images: a Clef with its vision tower and processor, on an unsloth-zoo that reads them."""
+def _mlx_reads(checkpoint: Checkpoint, media: str) -> bool:
+ """Whether the MLX form of an entry reads `media` ("images" or "audio"): a Clef with that tower and its processor, on an unsloth-zoo that reads it."""
if checkpoint.layout != "clef" or _mlx_target(checkpoint) is None:
return False
try:
from unsloth_zoo.mlx.decision import ClefModel
except Exception:
return False
- if not getattr(ClefModel, "takes_images", False):
+ if not getattr(ClefModel, f"takes_{media}", False):
return False
if not checkpoint.is_local:
- return True
+ return media == "images" or checkpoint.audio
import json
folder = Path(checkpoint.source).expanduser()
@@ -199,14 +202,24 @@ def _mlx_reads_images(checkpoint: Checkpoint) -> bool:
config = json.loads((folder / "config.json").read_text(encoding = "utf-8"))
except (OSError, ValueError):
return False
- return (
- isinstance(config, dict)
- and config.get("vision_config") is not None
- and any(
- (folder / name).is_file()
- for name in ("processor_config.json", "preprocessor_config.json")
- )
+ if not isinstance(config, dict):
+ return False
+ # Qwen3-Omni keeps its towers under the thinker, Phi-4-multimodal under `embd_layer`; Nemotron Omni hears through a sound tower.
+ parts = (config, config.get("thinker_config"), config.get("embd_layer"))
+ towers = (
+ ("vision_config", "image_embd_layer")
+ if media == "images"
+ else ("audio_config", "sound_config", "audio_embd_layer")
)
+ return any(
+ isinstance(part, dict) and part.get(tower) is not None for part in parts for tower in towers
+ ) and any(
+ (folder / name).is_file() for name in ("processor_config.json", "preprocessor_config.json")
+ )
+
+
+def _mlx_reads_images(checkpoint: Checkpoint) -> bool:
+ return _mlx_reads(checkpoint, "images")
def _mlx_repos(companion) -> tuple:
@@ -423,12 +436,14 @@ def select(
questions = None,
preference: str | None = None,
state_images: bool = False,
+ audio = None,
) -> tuple[Checkpoint, str | None]:
"""(what serves this request, why Auto did not pick llama.cpp); raises for what nothing here can serve.
Auto takes llama.cpp when the entry has a GGUF and Studio's llama-server serves decisions, else PyTorch;
on Apple Silicon it answers through unsloth-zoo's MLX engine first.
- Images are read by llama.cpp, and by MLX for a Clef. Laya is always served by PyTorch, a GGUF entry otherwise by llama.cpp.
+ Images are read by llama.cpp, and by MLX for a Clef. Audio is heard only by MLX, for a Clef with an audio tower.
+ Laya is always served by PyTorch, a GGUF entry otherwise by llama.cpp.
"""
from utils.systemone_settings import get_backend
@@ -436,6 +451,29 @@ def select(
# Only route-checked images reach MLX: state images would be decoded unvalidated in this process.
unread = bool(state_images or (images and not _mlx_reads_images(checkpoint)))
+ if audio:
+ # No other runtime hears, so nothing else may answer as if the clips were not sent.
+ if not _mlx_reads(checkpoint, "audio"):
+ raise Unavailable(
+ 400,
+ "api_usage_error",
+ f"{checkpoint.name} does not hear audio here; audio needs a Clef with an audio tower, served by MLX on Apple Silicon.",
+ )
+ if (preference or get_backend()) not in ("auto", MLX):
+ raise Unavailable(
+ 400,
+ "api_usage_error",
+ "Audio is served only by MLX; set the Decision API runtime to Auto or MLX.",
+ )
+ return _select_mlx(checkpoint, unread), None
+ if _is_mlx(checkpoint):
+ if (preference or get_backend()) not in ("auto", MLX):
+ raise Unavailable(
+ 400,
+ "api_usage_error",
+ f"{checkpoint.name} is served only by MLX; set the Decision API runtime to Auto or MLX.",
+ )
+ return _select_mlx(checkpoint, unread), None
if (preference or get_backend()) == MLX and checkpoint.layout in ("clef", GGUF):
return _select_mlx(checkpoint, unread), None
mlx = _mlx_choice(
@@ -501,6 +539,10 @@ def _select_mlx(checkpoint: Checkpoint, unread) -> Checkpoint:
return target
else:
reason = f"{checkpoint.name} has no MLX form."
+ if _is_mlx(checkpoint):
+ raise Unavailable(
+ 400, "api_usage_error", f"{checkpoint.name} is served only by MLX. {reason}"
+ )
raise Unavailable(400, "api_usage_error", f"{reason} Set the Decision API runtime to Auto.")
@@ -575,9 +617,15 @@ def native_ready(checkpoint) -> bool:
def input_modalities(checkpoint) -> list[str]:
- """What this entry reads here: images too when llama.cpp serves it with a vision projector, or MLX a Clef that has its tower."""
- if mlx_ready(checkpoint) and _mlx_reads_images(checkpoint):
- return ["text", "image"]
+ """What this entry reads here: images too when llama.cpp serves it with a vision projector, or MLX a Clef that has its tower; audio when MLX serves a Clef that hears."""
+ if mlx_ready(checkpoint):
+ media = [
+ kind
+ for kind, media in (("image", "images"), ("audio", "audio"))
+ if _mlx_reads(checkpoint, media)
+ ]
+ if media:
+ return ["text", *media]
if not native_ready(checkpoint):
return ["text"]
return ["text", "image"] if _native_reads_images(_native_target(checkpoint)) else ["text"]
@@ -940,6 +988,7 @@ class _EngineAgent:
state,
questions: dict[str, dict[str, Any]],
images = None,
+ audio = None,
) -> dict[str, Any]:
from unsloth_zoo.mlx.decision import DecisionRequestError, DecisionUnsupportedError
@@ -977,11 +1026,9 @@ class _EngineAgent:
f"The state is longer than the {limit} tokens this model reads beside a question. Shorten it.",
)
try:
- return (
- self.model.answer(state, questions, images)
- if images
- else self.model.answer(state, questions)
- )
+ # Only what was sent is passed, so an unsloth-zoo that takes less still answers text.
+ media = (images, audio) if audio else (images,) if images else ()
+ return self.model.answer(state, questions, *media)
except DecisionUnsupportedError as exc:
raise Unavailable(501, "not_supported_error", str(exc)) from None
except DecisionRequestError as exc:
@@ -2144,16 +2191,23 @@ def decide(
state,
questions: dict[str, dict[str, Any]],
images: list[str] | None = None,
+ audio: list[str] | None = None,
) -> dict[str, Any]:
if not _admission.acquire(blocking = False):
raise Unavailable(529, "overloaded", "System One is busy; retry shortly", retry_after = 1)
try:
- return _route(checkpoint, state, questions, images or None)
+ return _route(checkpoint, state, questions, images or None, audio or None)
finally:
_admission.release()
-def _route(checkpoint: Checkpoint, state, questions, images) -> dict[str, Any]:
+def _route(
+ checkpoint: Checkpoint,
+ state,
+ questions,
+ images,
+ audio = None,
+) -> dict[str, Any]:
global _fallback_reason
from utils.systemone_settings import get_backend
@@ -2166,11 +2220,11 @@ def _route(checkpoint: Checkpoint, state, questions, images) -> dict[str, Any]:
# A request, unlike a settings read, waits for device detection, so the first one already knows of MLX.
hardware.get_device()
state_images = _state_has_images(state)
- target, reason = select(checkpoint, images, questions, state_images = state_images)
+ target, reason = select(checkpoint, images, questions, state_images = state_images, audio = audio)
if checkpoint.layout != "laya":
_fallback_reason = reason
try:
- return _decide(target, state, questions, images)
+ return _decide(target, state, questions, images, audio)
except NativeContextOverflow as exc:
# Auto answers on PyTorch only what its longer window can hold; at equal windows it would refuse too.
from .catalog import clef_unsupported_reason
@@ -2212,6 +2266,7 @@ def _decide(
state,
questions: dict[str, dict[str, Any]],
images: list[str] | None = None,
+ audio: list[str] | None = None,
) -> dict[str, Any]:
global _last_used, _device_name
agent = _agent_for(checkpoint, LOAD_WAIT_S)
@@ -2227,7 +2282,7 @@ def _decide(
_last_used = time.monotonic()
return result
if _is_mlx(checkpoint):
- result = agent.decide(state, questions, images)
+ result = agent.decide(state, questions, images, audio)
_last_used = time.monotonic()
return {
"model": checkpoint.name,
diff --git a/studio/backend/core/training/decision_trainer.py b/studio/backend/core/training/decision_trainer.py
index 0ed6e88d02b..96f723cf788 100644
--- a/studio/backend/core/training/decision_trainer.py
+++ b/studio/backend/core/training/decision_trainer.py
@@ -129,6 +129,27 @@ def _read_local_rows(paths: list[str], load_dataset, warn: Callable[[str], None]
return [row for row in rows if isinstance(row, dict)]
+def _hold_audio(rows: list[dict]) -> None:
+ # The audio files an S3 download rewrote rows to name are deleted with it, before training reads them.
+ for row in rows:
+ audio = row.get("audio")
+ if audio is None:
+ continue
+ clips = audio if isinstance(audio, list) else [audio]
+ held = [_held_clip(clip) for clip in clips]
+ row["audio"] = held if isinstance(audio, list) else held[0]
+
+
+def _held_clip(clip):
+ # The rewrite leaves a plain path string or a {"path"} dict naming the download.
+ if isinstance(clip, str) and os.path.isabs(clip) and os.path.isfile(clip):
+ return {"bytes": Path(clip).read_bytes(), "path": clip}
+ if isinstance(clip, dict) and clip.get("bytes") is None and clip.get("path"):
+ if os.path.isfile(clip["path"]):
+ clip["bytes"] = Path(clip["path"]).read_bytes()
+ return clip
+
+
def _eval_sample(rows, seed: int):
# Every row holds at least one decision, so EVAL_MAX rows can fill the eval set.
if len(rows) <= EVAL_MAX:
@@ -161,6 +182,7 @@ def _load_rows(
raise _Stopped("Training cancelled") from None
try:
rows = _read_local_rows(download.files, load_dataset, warn)
+ _hold_audio(_sliced(rows, config))
finally:
download.cleanup()
elif config.get("hf_dataset"):
@@ -201,10 +223,14 @@ def _load_rows(
_read_local_rows(config["local_eval_datasets"], load_dataset, warn), seed
)
+ return _sliced(rows, config), eval_rows
+
+
+def _sliced(rows: list, config: dict) -> list:
start, end = config.get("dataset_slice_start"), config.get("dataset_slice_end")
- if start is not None or end is not None:
- rows = rows[start or 0 : (len(rows) if end is None else end + 1)]
- return rows, eval_rows
+ if start is None and end is None:
+ return rows
+ return rows[start or 0 : (len(rows) if end is None else end + 1)]
def _is_mlx() -> bool:
diff --git a/studio/backend/core/training/s3_dataset.py b/studio/backend/core/training/s3_dataset.py
index 709fe23404c..247f084efca 100644
--- a/studio/backend/core/training/s3_dataset.py
+++ b/studio/backend/core/training/s3_dataset.py
@@ -266,14 +266,21 @@ def _rewrite_row(row, resolve) -> None:
if not isinstance(row, dict):
return
for column, value in row.items():
- replacement = resolve(value)
+ row[column] = _rewritten(value, resolve)
+
+
+def _rewritten(value, resolve):
+ replacement = resolve(value)
+ if replacement is not None:
+ return replacement
+ if isinstance(value, dict):
+ # HF undecoded audio: {"path": ..., "bytes": ...}
+ replacement = resolve(value.get("path"))
if replacement is not None:
- row[column] = replacement
- elif isinstance(value, dict):
- # HF undecoded audio: {"path": ..., "bytes": ...}
- replacement = resolve(value.get("path"))
- if replacement is not None:
- value["path"] = replacement
+ value["path"] = replacement
+ elif isinstance(value, list):
+ return [_rewritten(item, resolve) for item in value]
+ return value
def _rewrite_json_manifest(manifest_path: str, resolve) -> None:
diff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.py
index dcab5fc7cef..dd85adf29ad 100644
--- a/studio/backend/core/training/training.py
+++ b/studio/backend/core/training/training.py
@@ -451,6 +451,8 @@ def _apply_model_cache_pin(config: dict[str, Any], warnings: list[str]) -> None:
config["actual_model_repo_id"] = None
config["model_snapshot_path"] = None
config["model_revision"] = None
+ # Nothing is pinned, so an offline start's pin has nothing left to validate.
+ config["require_validated_model_snapshot"] = False
return
requested_pin = config.get("model_snapshot_path")
require_validated_snapshot = bool(config.get("require_validated_model_snapshot"))
diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py
index 69a9723319e..e151712e61d 100644
--- a/studio/backend/routes/settings.py
+++ b/studio/backend/routes/settings.py
@@ -1609,13 +1609,15 @@ def _decision_description(checkpoint, mlx: bool) -> str:
def _clef_availability(checkpoint, reason: Optional[str]) -> dict:
from core.systemone import laya_runtime
- if getattr(checkpoint, "layout", "laya") == laya_runtime.GGUF:
+ only_mlx = getattr(checkpoint, "backend", None) == laya_runtime.MLX
+ if only_mlx or getattr(checkpoint, "layout", "laya") == laya_runtime.GGUF:
+ only = {} if only_mlx else {"llama_cpp_only": True}
# Selectable under a PyTorch runtime: the runtime row then says to switch it.
try:
laya_runtime.select(checkpoint, preference = "auto")
except laya_runtime.Unavailable as exc:
- return {"llama_cpp_only": True, "available": False, "unavailable_reason": exc.message}
- return {"llama_cpp_only": True}
+ return {**only, "available": False, "unavailable_reason": exc.message}
+ return only
if reason is None or getattr(checkpoint, "layout", "laya") != "clef":
return {}
# llama.cpp serves Clef without CUDA or ROCm, and so does the MLX engine on Apple Silicon.
diff --git a/studio/backend/routes/systemone.py b/studio/backend/routes/systemone.py
index d52e3d196e4..873676329f3 100644
--- a/studio/backend/routes/systemone.py
+++ b/studio/backend/routes/systemone.py
@@ -40,6 +40,9 @@ MAX_QUESTION_CHARS = 20_000
MAX_IMAGES = 4
MAX_IMAGE_BYTES = 4 * 1024 * 1024
MAX_IMAGES_BYTES = 8 * 1024 * 1024
+# As many clips as unsloth-zoo's MLX engine takes; 40 MiB holds the 20 minutes it hears as 16 kHz mono WAV.
+MAX_AUDIO = 8
+MAX_AUDIO_BYTES = 40 * 1024 * 1024
# What llama.cpp's mtmd decodes in-process (stb_image); WebP needs an ffmpeg it may not have.
_IMAGE_TYPES = {"image/png": "PNG", "image/jpeg": "JPEG"}
_TYPES = ("noul", "choice", "score")
@@ -69,6 +72,7 @@ class SystemOneRequest(BaseModel):
questions: dict[str, QuestionIn] = Field(min_length = 1)
# OpenJev's image extension: base64 data URLs, served by a Clef model through llama.cpp.
images: Optional[list[str]] = None
+ audio: Optional[list[str]] = None
def _error(
@@ -161,7 +165,9 @@ async def system_one(
)
if checkpoint is None:
raise _error(400, "api_usage_error", f"Unknown model: {payload.model}")
- result = await _decide(checkpoint, payload.state, payload.questions, payload.images)
+ result = await _decide(
+ checkpoint, payload.state, payload.questions, payload.images, payload.audio
+ )
headers = {"x-typesafe-request-id": str(uuid4())}
if isinstance(checkpoint, catalog.Checkpoint):
headers["x-unsloth-decision-backend"] = result.pop("_backend", "pytorch")
@@ -213,6 +219,35 @@ def _validate_images(images: list[str]) -> None:
)
+def _validate_audio(audio: list[str]) -> None:
+ # Shape and size only: the engine that hears the clips decodes them, and refuses what it cannot.
+ if len(audio) > MAX_AUDIO:
+ raise _error(422, "invalid_request_error", f"At most {MAX_AUDIO} audio clips per request")
+ total = 0
+ for index, url in enumerate(audio):
+ header, comma, data = url.partition(",")
+ kind = header.removeprefix("data:audio/").removesuffix(";base64")
+ if not comma or not kind or header != f"data:audio/{kind};base64":
+ raise _error(
+ 422,
+ "invalid_request_error",
+ f"audio[{index}] must be a base64 audio data URL; remote URLs are not fetched",
+ )
+ total += 3 * len(data) // 4
+ if not data or total > MAX_AUDIO_BYTES:
+ raise _error(
+ 422,
+ "invalid_request_error",
+ f"All audio clips together must be at most {MAX_AUDIO_BYTES // 1024**2} MiB",
+ )
+ try:
+ base64.b64decode(data, validate = True)
+ except (binascii.Error, ValueError):
+ raise _error(
+ 422, "invalid_request_error", f"audio[{index}] is not valid base64"
+ ) from None
+
+
MAX_IMAGE_PIXELS = 4096 * 4096
@@ -240,6 +275,7 @@ async def _decide(
state: JSONContent,
questions: dict[str, QuestionIn],
images: list[str] | None = None,
+ audio: list[str] | None = None,
) -> dict:
if not questions:
raise _error(422, "invalid_request_error", "At least one question is required")
@@ -260,6 +296,12 @@ async def _decide(
raise _error(
400, "api_usage_error", "Images are not forwarded to Decision API connections"
)
+ if audio:
+ _validate_audio(audio)
+ if isinstance(checkpoint, catalog.Connection):
+ raise _error(
+ 400, "api_usage_error", "Audio is not forwarded to Decision API connections"
+ )
if isinstance(checkpoint, catalog.Connection):
return await _connection_decide(
@@ -274,6 +316,7 @@ async def _decide(
state,
{name: q.model_dump() for name, q in questions.items()},
images or None,
+ audio or None,
)
except laya_runtime.Unavailable as exc:
raise _error(exc.status, exc.error_type, exc.message, exc.retry_after) from None
diff --git a/studio/backend/tests/test_decision_serving.py b/studio/backend/tests/test_decision_serving.py
index ce081f79660..9e8eae9995f 100644
--- a/studio/backend/tests/test_decision_serving.py
+++ b/studio/backend/tests/test_decision_serving.py
@@ -914,7 +914,7 @@ def test_clef_serves_on_nvidia_and_amd_gpus(home, client, clef, monkeypatch, kin
assert all(
m["available"] and m["unavailable_reason"] is None
for m in models
- if not m["llama_cpp_only"]
+ if not m["llama_cpp_only"] and m["name"] != "clef-omni"
)
@@ -1083,7 +1083,7 @@ def test_the_mlx_runtime_serves_only_through_the_engine(home, client, engine, mo
assert _put(client, device = "cpu").status_code == 200
settings = client.get("/api/settings/systemone").json()
assert not settings["mlx_available"] and settings["effective_backend"] is None
- assert not any("MLX" in m["description"] for m in settings["models"])
+ assert not any("or MLX" in m["description"] for m in settings["models"])
assert "needs Apple Silicon" in settings["fallback_reason"] and _post(client).status_code == 400
plan = client.get("/api/settings/systemone/resolve", params = {"backend": "mlx"}).json()
assert (plan["repo"], plan["cached"]) == (None, False) and "needs Apple Silicon" in plan[
@@ -1242,6 +1242,7 @@ def test_images_in_the_state_route_like_the_images_field(home, monkeypatch):
questions = None,
preference = None,
state_images = False,
+ audio = None,
):
seen.append(state_images and not images)
raise laya_runtime.Unavailable(400, "api_usage_error", "stop")
@@ -1336,6 +1337,105 @@ def test_a_clef_with_its_vision_tower_reads_images_through_the_mlx_engine(
assert engine.read == ("s", ())
+def test_a_clef_with_its_audio_tower_hears_through_the_mlx_engine(
+ home, client, engine, monkeypatch
+):
+ served, folder = _clef_fine_tune(home, "clef_mlx_4"), home / "clef_mlx_4"
+ tuned, kev = catalog.fine_tune(served), catalog.CHECKPOINTS["kev-4b"]
+ zoo, clip = sys.modules["unsloth_zoo.mlx.decision"], "data:audio/wav;base64,AA=="
+ image, questions = "data:image/png;base64,AA==", {"q": {"type": "noul", "instructions": "i"}}
+ (folder / "processor_config.json").write_text("{}", encoding = "utf-8")
+
+ def hears(checkpoint):
+ try:
+ routed = laya_runtime.select(checkpoint, audio = [clip])[0].backend == "mlx"
+ except laya_runtime.Unavailable as refused:
+ routed = "does not hear audio here" not in refused.message
+ assert routed == ("audio" in laya_runtime.input_modalities(checkpoint))
+ return routed
+
+ # An audio tower in the folder and an unsloth-zoo that hears: neither alone, and no other entry.
+ zoo.ClefModel = SimpleNamespace(takes_images = True, takes_audio = True)
+ (folder / "config.json").write_text('{"vision_config": {}}', encoding = "utf-8")
+ assert not hears(tuned)
+ omni = {"thinker_config": {"audio_config": {}, "vision_config": {}}}
+ (folder / "config.json").write_text(json.dumps(omni), encoding = "utf-8")
+ assert hears(tuned) and not hears(kev) and not hears(catalog.CHECKPOINTS["clef-flash"])
+ assert laya_runtime.input_modalities(tuned) == ["text", "image", "audio"]
+ # Phi-4-multimodal and Nemotron Omni name their towers otherwise.
+ phi4 = {"embd_layer": {"image_embd_layer": {}, "audio_embd_layer": {}}}
+ for config, modalities in (
+ (phi4, ["text", "image", "audio"]),
+ ({"vision_config": {}, "sound_config": {}}, ["text", "image", "audio"]),
+ ):
+ (folder / "config.json").write_text(json.dumps(config), encoding = "utf-8")
+ assert hears(tuned) and laya_runtime.input_modalities(tuned) == modalities
+ (folder / "config.json").write_text(
+ json.dumps({"embd_layer": {"audio_embd_layer": {}}}), encoding = "utf-8"
+ )
+ # A model that hears but does not see still reports the audio it is routed.
+ assert hears(tuned) and laya_runtime.input_modalities(tuned) == ["text", "audio"]
+ (folder / "config.json").write_text(json.dumps(omni), encoding = "utf-8")
+ zoo.ClefModel = SimpleNamespace(takes_images = True)
+ assert not hears(tuned)
+ zoo.ClefModel = SimpleNamespace(takes_images = True, takes_audio = True)
+ for runtime in ("pytorch", "llama.cpp"):
+ with pytest.raises(laya_runtime.Unavailable, match = "Audio is served only by MLX"):
+ laya_runtime.select(tuned, audio = [clip], preference = runtime)
+ assert _put(client, enabled = True, model = served).status_code == 200
+ clips = [clip, "data:audio/wav;base64,AQ=="]
+ body = {"model": "default", "state": "s", "questions": questions, "audio": clips}
+ assert client.post("/v1/systemone", json = body).headers["x-unsloth-decision-backend"] == "mlx"
+ assert engine.read == ("s", (None, clips))
+ assert client.post("/v1/systemone", json = {**body, "model": "kev-4b"}).status_code == 400
+ assert laya_runtime._route(tuned, "s", questions, [image], clips)["_backend"] == "mlx"
+ assert engine.read == ("s", ([image], clips))
+ monkeypatch.setattr(systemone, "MAX_AUDIO_BYTES", 6)
+ for audio, message in (
+ ([clip] * 9, "At most 8 audio clips"),
+ ([clip, image], "audio[1] must be a base64 audio data URL"),
+ ([clip, "data:audio/wav;base64,@@@@"], "audio[1] is not valid base64"),
+ ([clip] * 3, "All audio clips together"),
+ ):
+ refused = client.post("/v1/systemone", json = {**body, "audio": audio})
+ assert refused.status_code == 422 and message in refused.json()["detail"]["message"]
+ asked, connection = {"q": systemone.QuestionIn(type = "noul")}, catalog.Connection("p", "m")
+ with pytest.raises(systemone.HTTPException, match = "Audio is not forwarded"):
+ asyncio.run(systemone._decide(connection, "s", asked, None, [clip]))
+
+
+def test_clef_omni_is_served_by_the_mlx_engine_or_not_at_all(home, client, engine, monkeypatch):
+ omni, clip = catalog.CHECKPOINTS["clef-omni"], "data:audio/wav;base64,AA=="
+ zoo = sys.modules["unsloth_zoo.mlx.decision"]
+ zoo.ClefModel = SimpleNamespace(takes_images = True, takes_audio = True)
+ assert _put(client, enabled = True, model = "clef-omni").status_code == 200
+ settings = client.get("/api/settings/systemone").json()
+ assert settings["effective_backend"] == "mlx"
+ assert settings["input_modalities"] == ["text", "image", "audio"]
+ questions = {"q": {"type": "noul", "instructions": "i"}}
+ body = {"model": "default", "state": "s", "questions": questions, "audio": [clip]}
+ assert client.post("/v1/systemone", json = body).headers["x-unsloth-decision-backend"] == "mlx"
+ assert engine.loaded == [("clef-omni-8bit", "clef", None)] and engine.read[1] == (None, [clip])
+ assert not laya_runtime.input_modalities(catalog.CHECKPOINTS["clef"])[2:]
+ # It needs no PyTorch, so a --no-torch install on Apple Silicon enables and lists it.
+ with monkeypatch.context() as patch:
+ patch.setattr(systemone_settings, "runtime_unavailable_reason", lambda: "no torch")
+ systemone_settings.validate(enabled = True, model = "clef-omni")
+ with pytest.raises(ValueError, match = "no torch"):
+ systemone_settings.validate(enabled = True, model = "clef")
+ listed = [m["id"] for m in systemone.decision_model_objects()]
+ assert "clef-omni" in listed and "clef" not in listed
+ for runtime in ("pytorch", "llama.cpp"):
+ with pytest.raises(laya_runtime.Unavailable, match = "clef-omni is served only by MLX"):
+ laya_runtime.select(omni, preference = runtime)
+ # Where the engine is not, the entry is listed as one this machine cannot serve.
+ _spoof_device(monkeypatch, "cuda")
+ listed = {m["name"]: m for m in client.get("/api/settings/systemone").json()["models"]}
+ assert listed["clef-omni"]["available"] is False and not listed["clef-omni"]["llama_cpp_only"]
+ assert "served only by MLX" in listed["clef-omni"]["unavailable_reason"]
+ assert _post(client, "clef-omni").status_code == 400
+
+
def test_auto_keeps_images_in_the_state_off_mlx(home, engine, monkeypatch):
kev = catalog.CHECKPOINTS["kev-4b"]
monkeypatch.setattr(laya_runtime, "_native_unavailable", lambda *args: None)
diff --git a/studio/backend/tests/test_decision_training.py b/studio/backend/tests/test_decision_training.py
index f94afd6c0a9..d2172ea48d9 100644
--- a/studio/backend/tests/test_decision_training.py
+++ b/studio/backend/tests/test_decision_training.py
@@ -758,6 +758,51 @@ def test_a_stop_during_the_s3_download_cancels_the_run(monkeypatch):
_load(lambda: True, s3_config = {"bucket": "b"})
+def test_s3_audio_rows_keep_their_clips_after_the_download_is_cleaned_up(monkeypatch, tmp_path):
+ import shutil
+ from types import SimpleNamespace
+
+ from core.training import s3_dataset
+
+ clips = [tmp_path / "a.wav", tmp_path / "b.wav", tmp_path / "c.wav"]
+ for index, clip in enumerate(clips):
+ clip.write_bytes(b"RIFF" + bytes([index]))
+ manifest = tmp_path / "rows.jsonl"
+ rows = [
+ {**_row(0), "audio": {"bytes": None, "path": str(clips[0])}},
+ {**_row(1), "audio": [{"bytes": None, "path": str(clips[1])}]},
+ # A plain path string, as the S3 rewrite leaves a manifest's usual audio cell.
+ {**_row(2), "audio": str(clips[2])},
+ _row(3),
+ ]
+ manifest.write_text("\n".join(json.dumps(row) for row in rows), encoding = "utf-8")
+ copy = tmp_path / "download"
+ copy.mkdir()
+ shutil.copy(manifest, copy)
+
+ def download(s3_config, cancel_callback):
+ return SimpleNamespace(
+ files = [str(copy / "rows.jsonl")], cleanup = lambda: [clip.unlink() for clip in clips]
+ )
+
+ monkeypatch.setattr(s3_dataset, "prepare_s3_dataset_download", download)
+ loaded, _, _ = _load(s3_config = {"bucket": "b"})
+ assert [
+ loaded[0]["audio"]["bytes"],
+ loaded[1]["audio"][0]["bytes"],
+ loaded[2]["audio"]["bytes"],
+ ] == [b"RIFF\x00", b"RIFF\x01", b"RIFF\x02"]
+ assert "audio" not in loaded[3]
+ assert not any(clip.exists() for clip in clips)
+ # A slice reads only the clips of the rows it keeps.
+ for index, clip in enumerate(clips):
+ clip.write_bytes(b"RIFF" + bytes([index]))
+ read, read_bytes = [], Path.read_bytes
+ monkeypatch.setattr(Path, "read_bytes", lambda path: read.append(path.name) or read_bytes(path))
+ loaded, _, _ = _load(s3_config = {"bucket": "b"}, dataset_slice_start = 1, dataset_slice_end = 1)
+ assert [row["audio"][0]["bytes"] for row in loaded] == [b"RIFF\x01"] and read == ["b.wav"]
+
+
def test_other_accounts_reach_only_the_configured_fine_tune(studio_home, client):
from utils.account_context import AccountContext, bind_account, reset_account
from utils.paths import outputs_root
diff --git a/studio/backend/tests/test_decision_training_route.py b/studio/backend/tests/test_decision_training_route.py
index b7bd8d323fe..79d80114d65 100644
--- a/studio/backend/tests/test_decision_training_route.py
+++ b/studio/backend/tests/test_decision_training_route.py
@@ -497,6 +497,30 @@ def test_a_hub_checkpoint_downloads_under_the_stall_watchdog(monkeypatch, tmp_pa
assert events == [] and len(downloads) == 1
+def test_an_offline_decision_start_does_not_wait_for_a_pin_it_never_uses(tmp_path):
+ from core.training.training import _apply_cache_pins
+ from core.training.worker import _verify_config_pins
+
+ events = []
+ queue = SimpleNamespace(put = events.append)
+ # Offline, the start route pins the cached snapshot of an LLM given a decision head, as of any LLM.
+ snapshot = str(tmp_path / "snapshot")
+ pinned = {
+ "model_name": "unsloth/Qwen3.5-0.8B",
+ "actual_model_repo_id": "unsloth/Qwen3.5-0.8B",
+ "model_snapshot_path": snapshot,
+ "require_validated_model_snapshot": True,
+ }
+ decision = {**pinned, "is_decision": True, "decision_layout": "llm"}
+ _apply_cache_pins(decision)
+ assert _verify_config_pins(decision, queue) and events == []
+ # An LLM run still loads exactly the snapshot it was pinned to, or not at all.
+ from core.training.provenance import ExactResumeResourcesUnavailable
+
+ with pytest.raises(ExactResumeResourcesUnavailable, match = "no longer available"):
+ _apply_cache_pins(pinned)
+
+
@pytest.mark.skipif(sys.platform == "darwin", reason = "Apple Silicon trains with MLX")
def test_decision_worker_uses_one_gpu_and_no_llm_kernel_installs(monkeypatch, tmp_path):
import multiprocessing as mp
diff --git a/studio/backend/tests/test_s3_audio_dataset.py b/studio/backend/tests/test_s3_audio_dataset.py
index e3488538adf..39cc07eb477 100644
--- a/studio/backend/tests/test_s3_audio_dataset.py
+++ b/studio/backend/tests/test_s3_audio_dataset.py
@@ -148,6 +148,7 @@ def test_jsonl_audio_references_are_rewritten_to_local_paths(monkeypatch, tmp_pa
json.dumps({"audio": {"path": "audio/a.wav"}, "text": "three"}),
# Unmatched references are somebody else's contract: left alone.
json.dumps({"audio": "https://example.com/c.wav", "text": "four"}),
+ json.dumps({"audio": [{"path": "audio/a.wav"}, "audio/b.mp3"], "text": "five"}),
]
)
_install(
@@ -165,6 +166,7 @@ def test_jsonl_audio_references_are_rewritten_to_local_paths(monkeypatch, tmp_pa
assert rows[1]["audio"] == local_b
assert rows[2]["audio"]["path"] == local_a
assert rows[3]["audio"] == "https://example.com/c.wav"
+ assert rows[4]["audio"] == [{"path": local_a}, local_b]
assert all(os.path.isabs(p) for p in (rows[0]["audio"], rows[1]["audio"]))
assert rows[0]["text"] == "one"
diff --git a/studio/backend/utils/systemone_settings.py b/studio/backend/utils/systemone_settings.py
index 6a1632a5fe8..77e3f9428c1 100644
--- a/studio/backend/utils/systemone_settings.py
+++ b/studio/backend/utils/systemone_settings.py
@@ -71,9 +71,11 @@ def runtime_unavailable_reason() -> str | None:
def llama_cpp_only(name: Any) -> bool:
- """A GGUF-only catalog entry: llama.cpp serves it without PyTorch."""
+ """A catalog entry served without PyTorch: a GGUF by llama.cpp, an MLX-only entry by unsloth-zoo's MLX engine."""
from core.systemone.catalog import CHECKPOINTS
- return getattr(CHECKPOINTS.get(name), "layout", None) == "gguf"
+
+ entry = CHECKPOINTS.get(name)
+ return getattr(entry, "layout", None) == "gguf" or getattr(entry, "backend", None) == "mlx"
def get_enabled() -> bool:
diff --git a/studio/frontend/src/config/training.ts b/studio/frontend/src/config/training.ts
index f86d415a46e..003ff34f84f 100644
--- a/studio/frontend/src/config/training.ts
+++ b/studio/frontend/src/config/training.ts
@@ -232,10 +232,29 @@ export const PRIORITY_TRAINING_MODELS: readonly string[] = [
"unsloth/Llama-3.2-3B-Instruct",
];
+const LAYA_TRAINING_MODEL = "convaiinnovations/laya";
+const CLEF_OMNI_MLX_TRAINING_MODEL = "mlx-community/clef-omni-8bit";
+
+/** The pinned models for a device: Clef Omni trains only through MLX, so only a Mac lists it. */
+export function priorityTrainingModels(
+ deviceType?: string | null,
+): readonly string[] {
+ return deviceType === "mac"
+ ? PRIORITY_TRAINING_MODELS.flatMap((id) =>
+ id === LAYA_TRAINING_MODEL ? [id, CLEF_OMNI_MLX_TRAINING_MODEL] : [id],
+ )
+ : PRIORITY_TRAINING_MODELS;
+}
+
/** Pin priority models to the top of a list of model IDs, preserving their defined order. */
-export function applyPriorityOrdering(ids: string[]): string[] {
+export function applyPriorityOrdering(
+ ids: string[],
+ deviceType?: string | null,
+): string[] {
const idSet = new Set(ids);
- const pinned = PRIORITY_TRAINING_MODELS.filter((id) => idSet.has(id));
+ const pinned = priorityTrainingModels(deviceType).filter((id) =>
+ idSet.has(id),
+ );
const pinnedSet = new Set(pinned);
const rest = ids.filter((id) => !pinnedSet.has(id));
return [...pinned, ...rest];
diff --git a/studio/frontend/src/features/settings/lib/decision-model-labels.ts b/studio/frontend/src/features/settings/lib/decision-model-labels.ts
index b2a3c37a7c5..9c855a0f42f 100644
--- a/studio/frontend/src/features/settings/lib/decision-model-labels.ts
+++ b/studio/frontend/src/features/settings/lib/decision-model-labels.ts
@@ -14,6 +14,9 @@ export const DECISION_MODEL_LABELS: Record<string, TranslationKey> = {
/** Cloudflare Clef models are named by themselves; the Laya labels read as "Laya <label>". */
export function isClefDecisionModel(name: string | null | undefined): boolean {
return (
- name === "clef" || name === "clef-flash" || !!name?.startsWith("clef-ft:")
+ name === "clef" ||
+ name === "clef-flash" ||
+ name === "clef-omni" ||
+ !!name?.startsWith("clef-ft:")
);
}
diff --git a/studio/frontend/src/features/train-model-picker/components/train-model-selector.tsx b/studio/frontend/src/features/train-model-picker/components/train-model-selector.tsx
index 2ce2bb1d3e1..4882e58f898 100644
--- a/studio/frontend/src/features/train-model-picker/components/train-model-selector.tsx
+++ b/studio/frontend/src/features/train-model-picker/components/train-model-selector.tsx
@@ -18,8 +18,8 @@ import { usePickerState } from "@/components/resource-picker/use-picker-state";
import { usePlatformStore } from "@/config/env";
import {
MODEL_TYPE_TO_HF_TASKS,
- PRIORITY_TRAINING_MODELS,
applyPriorityOrdering,
+ priorityTrainingModels,
} from "@/config/training";
import {
type CachedInventoryRow,
@@ -509,7 +509,7 @@ export function TrainModelSelector({
task: trainingModelTaskFilter(requiredModelType),
accessToken: hfApiToken(picker.debouncedHfToken),
excludeGguf: true,
- priorityIds: PRIORITY_TRAINING_MODELS,
+ priorityIds: priorityTrainingModels(deviceType),
ownerScope: picker.debouncedHubQuery.trim() ? "all" : "unsloth",
enabled: online && picker.open && tab === PICKER_TAB.hub,
});
@@ -534,8 +534,8 @@ export function TrainModelSelector({
) {
ids.push(selectedModel);
}
- return applyPriorityOrdering(ids);
- }, [hfResults, modelKnownCached, modelLocalPath, selectedModel]);
+ return applyPriorityOrdering(ids, deviceType);
+ }, [deviceType, hfResults, modelKnownCached, modelLocalPath, selectedModel]);
const hfResultById = useMemo(() => {
const map = new Map<string, HfModelResult>();
diff --git a/studio/frontend/src/features/training/api/mappers.ts b/studio/frontend/src/features/training/api/mappers.ts
index 3e4318dd732..3dc40a6d2e3 100644
--- a/studio/frontend/src/features/training/api/mappers.ts
+++ b/studio/frontend/src/features/training/api/mappers.ts
@@ -304,7 +304,8 @@ export function buildTrainingStartPayload(
finetune_attention_modules: config.finetuneAttentionModules,
finetune_mlp_modules: config.finetuneMLPModules,
is_dataset_image: isEmbedding ? false : !!config.isDatasetImage,
- is_dataset_audio: isEmbedding ? false : config.isDatasetAudio,
+ // The decision trainer reads its rows itself; this flag selects the chat audio path.
+ is_dataset_audio: isEmbedding || isDecision ? false : config.isDatasetAudio,
is_embedding: isEmbedding && !isDecision,
is_decision: isDecision,
model_subfolder: isDecision ? config.modelSubfolder : null,
diff --git a/studio/frontend/src/features/training/hooks/use-training-readiness.ts b/studio/frontend/src/features/training/hooks/use-training-readiness.ts
index cd0d1153064..e775342dc13 100644
--- a/studio/frontend/src/features/training/hooks/use-training-readiness.ts
+++ b/studio/frontend/src/features/training/hooks/use-training-readiness.ts
@@ -45,7 +45,8 @@ export function deriveTrainingReadiness(
state.modelType === "decision" && modelError !== null;
const isModelCapabilitiesSettled = hasModel && !isLoadingModel;
const isIncompatible =
- isModelCapabilitiesSettled && hasIncompatibleTrainingModalities(state);
+ isModelCapabilitiesSettled &&
+ hasIncompatibleTrainingModalities(state, deviceType);
const hasDatasetModalityMetadata = state.isDatasetImage !== null;
const modelHandlesAllModalities = state.isVisionModel && state.isAudioModel;
const datasetUnverified =
diff --git a/studio/frontend/src/features/training/lib/model-type-inference.ts b/studio/frontend/src/features/training/lib/model-type-inference.ts
index b289d430b07..1ac1156ab0f 100644
--- a/studio/frontend/src/features/training/lib/model-type-inference.ts
+++ b/studio/frontend/src/features/training/lib/model-type-inference.ts
@@ -9,7 +9,7 @@ import {
import type { ModelTypeCapabilityFlags } from "./model-type-capabilities";
const TEXT_MODEL_TAGS = new Set(["text-generation"]);
-const DECISION_MODEL_TAGS = new Set(["laya"]);
+const DECISION_MODEL_TAGS = new Set(["laya", "clef"]);
export type TrainingModelTypeMetadata = TrainingModelModalityMetadata;
diff --git a/studio/frontend/src/features/training/lib/validation.ts b/studio/frontend/src/features/training/lib/validation.ts
index 0856ac9c6d3..7ee328e946d 100644
--- a/studio/frontend/src/features/training/lib/validation.ts
+++ b/studio/frontend/src/features/training/lib/validation.ts
@@ -44,11 +44,24 @@ function isTrainingConfigSupportedOnDevice(
if (deviceType !== "mac") {
return true;
}
- return !(config.isEmbeddingModel || config.isDatasetAudio === true);
+ return !(config.isEmbeddingModel || hasChatAudioRows(config, deviceType));
+}
+
+/** A Mac Clef (or Clef-head LLM) trainer reads its own audio rows; elsewhere, and for Laya, they train as text. */
+function hasChatAudioRows(
+ config: TrainingConfigState,
+ deviceType?: string,
+): boolean {
+ const trainerReadsRows =
+ deviceType === "mac" &&
+ config.modelType === "decision" &&
+ (config.decisionLayout === "clef" || config.decisionLayout === "llm");
+ return config.isDatasetAudio === true && !trainerReadsRows;
}
export function hasIncompatibleTrainingModalities(
config: TrainingConfigState,
+ deviceType?: string,
): boolean {
return (
(!config.isVisionModel && config.isDatasetImage === true) ||
@@ -58,7 +71,7 @@ export function hasIncompatibleTrainingModalities(
// real download error, which is what a gated text model already does.
(!config.audioCapabilityUnknown &&
!config.isAudioModel &&
- config.isDatasetAudio === true)
+ hasChatAudioRows(config, deviceType))
);
}
diff --git a/studio/frontend/tests/training-decision-model.test.ts b/studio/frontend/tests/training-decision-model.test.ts
index 1d456f09807..27c2c10183e 100644
--- a/studio/frontend/tests/training-decision-model.test.ts
+++ b/studio/frontend/tests/training-decision-model.test.ts
@@ -14,8 +14,13 @@ registerStoreStubResolver();
installLocalStorageFake();
const yaml = await import("js-yaml");
-const { DEFAULT_HYPERPARAMS, LR_DEFAULT_DECISION_FULL, LR_DEFAULT_LORA } =
- await import("../src/config/training.ts");
+const {
+ DEFAULT_HYPERPARAMS,
+ LR_DEFAULT_DECISION_FULL,
+ LR_DEFAULT_LORA,
+ applyPriorityOrdering,
+ priorityTrainingModels,
+} = await import("../src/config/training.ts");
const { setAuthFetchHandler } = await import("./helpers/store-stubs/auth.ts");
const { useTrainingConfigStore } = await import(
"../src/features/training/stores/training-config-store.ts"
@@ -29,6 +34,9 @@ const { DatasetFormatError } = await import(
const { validateTrainingConfig } = await import(
"../src/features/training/lib/validation.ts"
);
+const { classifyUnslothSupport } = await import(
+ "../src/features/hub/lib/unsloth-support.ts"
+);
const { deriveTrainingReadiness } = await import(
"../src/features/training/hooks/use-training-readiness.ts"
);
@@ -527,6 +535,65 @@ test("the model picker knows Laya is a decision model before its config loads",
assert.equal(isTrainingModelTypeSupportedOnDevice(modelType, "cuda"), true);
});
+test("a Mac lists Clef Omni and trains it on rows that carry audio", () => {
+ const omni = "mlx-community/clef-omni-8bit";
+ const metadata = {
+ pipelineTag: "zero-shot-classification",
+ tags: ["mlx", "safetensors", "qwen3_omni_moe", "clef", "audio", "video"],
+ };
+ assert.equal(
+ inferTrainingModelTypeFromFlags(
+ trainingModelTypeFlagsFromMetadata({
+ pipelineTag: "image-text-to-text",
+ tags: ["clef", "text-to-audio"],
+ }),
+ ),
+ "decision",
+ );
+ assert.equal(
+ inferTrainingModelTypeFromFlags(trainingModelTypeFlagsFromMetadata(metadata)),
+ "decision",
+ );
+ const support = classifyUnslothSupport({
+ modelId: omni,
+ libraryName: "mlx",
+ deviceType: "mac",
+ ...metadata,
+ });
+ assert.equal(support.status, "supported");
+ assert.equal(priorityTrainingModels("cuda").includes(omni), false);
+ assert.deepEqual(applyPriorityOrdering(["unsloth/x", omni, LAYA], "mac"), [
+ LAYA,
+ omni,
+ "unsloth/x",
+ ]);
+
+ const config = {
+ ...useTrainingConfigStore.getState(),
+ ...HF_DECISION_DATASET,
+ selectedModel: omni,
+ modelType: "decision" as const,
+ decisionLayout: "clef" as const,
+ isDatasetAudio: true,
+ };
+ assert.equal(deriveTrainingReadiness(config, "mac", true).isReady, true);
+ // So can an LLM that is given a decision head, such as a plain omni model.
+ const fresh = { ...config, decisionLayout: "llm" as const };
+ assert.equal(deriveTrainingReadiness(fresh, "mac", true).isReady, true);
+ // The backend refuses the chat audio flag on a Mac, and the decision trainer never reads it.
+ assert.equal(buildTrainingStartPayload(config, null).is_dataset_audio, false);
+ // Only the MLX Clef trainer reads the clips: Laya, another device and a chat model would not.
+ for (const [other, device] of [
+ [{ decisionLayout: "laya" as const }, "mac"],
+ [{}, "cuda"],
+ [{ modelType: "text" as const }, "mac"],
+ ] as const) {
+ const blocked = deriveTrainingReadiness({ ...config, ...other }, device, true);
+ assert.equal(blocked.isReady, false);
+ assert.equal(blocked.isIncompatible, true);
+ }
+});
+
test("only the Studio owner can start a decision run", () => {
const config = {
...useTrainingConfigStore.getState(),
diff --git a/tests/test_decision_model_mlx.py b/tests/test_decision_model_mlx.py
index 48ff1de000c..40d4b671127 100644
--- a/tests/test_decision_model_mlx.py
+++ b/tests/test_decision_model_mlx.py
@@ -435,12 +435,18 @@ def test_the_images_of_a_row_reach_the_clef_trainer_and_predict(
seen, real, keep = [], zoo.clef_training_item, [True]
- def item(pipeline, state, questions, max_length, *images):
- # A row without images is asked as before, with no fifth argument.
- seen.append([getattr(image, "size", image) for image in images[0]] if images else None)
+ def item(
+ pipeline,
+ state,
+ questions,
+ max_length,
+ images = None,
+ audio = None,
+ ):
+ seen.append([getattr(image, "size", image) for image in images] if images else audio)
return {
**real(pipeline, state, questions, max_length),
- **({"images": images[0]} if keep and images else {}),
+ **({"images": images} if keep and images else {}),
}
monkeypatch.setattr(zoo, "clef_training_item", item)
@@ -465,6 +471,40 @@ def test_the_images_of_a_row_reach_the_clef_trainer_and_predict(
model, tokenizer, "s", {"q": {"type": "noul", "instructions": "i"}}, images = url
)
assert seen[-1] == [url]
+ # Audio rows: data URLs as they are, an undecoded column's bytes or file as one, a decoded clip with its rate.
+ (tmp_path / "a.wav").write_bytes(b"RIFF")
+ sound = "data:audio/wav;base64," + base64.b64encode(b"RIFF").decode()
+ clip = {"path": None, "array": [0.0, 1.0], "sampling_rate": 16000}
+
+ class Decoder:
+ # What datasets 4 decodes an audio column to.
+ get_all_samples = None
+
+ def __getitem__(self, key):
+ return clip[key]
+
+ class Corrupt(Decoder):
+ def __getitem__(self, key):
+ raise RuntimeError("invalid data found")
+
+ heard = [
+ {**_row(0), "audio": [sound, {"bytes": b"RIFF"}]},
+ {**_row(1), "audio": {"bytes": None, "path": str(tmp_path / "a.wav")}},
+ {**_row(2), "audio": [None, {"bytes": None, "path": None}, clip, Decoder()]},
+ {**_row(3), "audio": {"bytes": None, "path": str(tmp_path / "none.wav")}},
+ ]
+ items, report = FastDecisionModel.build_dataset(
+ [_row(0), {**_row(1), "audio": [Decoder(), Corrupt()]}], tokenizer, model
+ )
+ assert [item["row"] for item in items] == [0] and "could not be decoded" in report["reason"]
+ del seen[:]
+ items, report = FastDecisionModel.build_dataset(heard, tokenizer, model)
+ assert [item["row"] for item in items] == [0, 1, 2] and "none.wav" in report["reason"]
+ assert seen == [[sound, sound], [sound], [{"array": [0.0, 1.0], "sampling_rate": 16000}] * 2]
+ FastDecisionModel.predict(
+ model, tokenizer, "s", {"q": {"type": "noul", "instructions": "i"}}, audio = sound
+ )
+ assert seen[-1] == [sound]
# An unsloth-zoo that takes no images says so, where skipping the rows would train without them.
monkeypatch.setattr(
zoo,
@@ -477,7 +517,9 @@ def test_the_images_of_a_row_reach_the_clef_trainer_and_predict(
question = {"q": {"type": "noul", "instructions": "i"}}
for call in (
lambda: FastDecisionModel.build_dataset(rows, tokenizer, model),
+ lambda: FastDecisionModel.build_dataset(heard, tokenizer, model),
lambda: FastDecisionModel.predict(model, tokenizer, "s", question, images = url),
+ lambda: FastDecisionModel.predict(model, tokenizer, "s", question, audio = sound),
):
with pytest.raises(ImportError, match = "newer unsloth-zoo"):
call()
@@ -486,6 +528,8 @@ def test_the_images_of_a_row_reach_the_clef_trainer_and_predict(
FastDecisionModel.predict(
laya, laya_tokenizer, "s", {"q": {"type": "noul", "instructions": "i"}}, images = [url]
)
+ with pytest.raises(unsloth._decision_mlx.DecisionDataError, match = "Laya reads text only"):
+ FastDecisionModel.predict(laya, laya_tokenizer, "s", question, audio = sound)
assert FastDecisionModel.predict(
laya, laya_tokenizer, "s", {"q": {"type": "noul", "instructions": "i"}}, images = "[]"
)
diff --git a/unsloth/models/decision_mlx.py b/unsloth/models/decision_mlx.py
index 69cd854ae21..ea77caf6ebc 100644
--- a/unsloth/models/decision_mlx.py
+++ b/unsloth/models/decision_mlx.py
@@ -58,9 +58,37 @@ from unsloth._decision_common import (
)
+def _row_audio(row: dict) -> list:
+ # Data URLs, decoded {"array", "sampling_rate"} 16 kHz clips, or an undecoded column's {"bytes", "path"}.
+ audio = _parsed(row.get("audio"))
+ if audio is None:
+ return []
+ found = []
+ for clip in audio if isinstance(audio, (list, tuple)) else [audio]:
+ if hasattr(clip, "get_all_samples"):
+ # datasets 4 decodes an audio column to a decoder that reads like the dict; a clip it cannot decode fails its row.
+ try:
+ clip = {"array": clip["array"], "sampling_rate": clip["sampling_rate"]}
+ except RuntimeError as exc:
+ raise ValueError(f"an audio clip could not be decoded: {exc}") from None
+ elif isinstance(clip, dict) and "array" in clip:
+ clip = {"array": clip["array"], "sampling_rate": clip.get("sampling_rate")}
+ elif isinstance(clip, dict) and set(clip) <= {"bytes", "path"}:
+ import base64
+
+ data = clip.get("bytes")
+ if data is None and clip.get("path"):
+ data = Path(clip["path"]).read_bytes()
+ # The container is recognized from the bytes, whatever the URL calls it.
+ clip = data and "data:audio/wav;base64," + base64.b64encode(data).decode()
+ if clip:
+ found.append(clip)
+ return found
+
+
def _decision_clef_items(rows, options: Callable, encode: Callable, validate, report, skip) -> list:
# One item per row. `options(question)` names a question's options in the head's order and
- # `encode(state, questions, images)` returns the backend's tokenized fields, raising when they do not fit.
+ # `encode(state, questions, images, audio)` returns the backend's tokenized fields, raising when they do not fit.
items = []
for index, row in enumerate(rows):
row = row if isinstance(row, dict) else {}
@@ -94,7 +122,7 @@ def _decision_clef_items(rows, options: Callable, encode: Callable, validate, re
if not kept:
continue
try:
- encoded = encode(state, kept, _row_images(row))
+ encoded = encode(state, kept, _row_images(row), _row_audio(row))
except (TypeError, ValueError, OSError) as exc:
for name in kept:
skip(index, f"does not fit: {exc}", name)
@@ -415,16 +443,25 @@ def _decision_pad_token_id(tokenizer) -> int:
return getattr(tokenizer, "tokenizer", tokenizer).pad_token_id
-def _clef_item(zoo, pipeline, state, questions, max_length, images):
- if not images:
- return zoo.clef_training_item(pipeline, state, questions, max_length)
- if "images" not in inspect.signature(zoo.clef_training_item).parameters:
- # Left to fail as a row that does not fit, a dataset would train without its image rows.
- raise ImportError(
- "Unsloth: images in decision rows need a newer unsloth-zoo. "
- "Upgrade with `pip install -U unsloth-zoo`."
- )
- return zoo.clef_training_item(pipeline, state, questions, max_length, images)
+def _clef_item(
+ zoo,
+ pipeline,
+ state,
+ questions,
+ max_length,
+ images,
+ audio = (),
+):
+ media = {name: value for name, value in (("images", images), ("audio", audio)) if value}
+ taken = inspect.signature(zoo.clef_training_item).parameters
+ for name in media:
+ if name not in taken:
+ # Left to fail as a row that does not fit, a dataset would train without these rows.
+ raise ImportError(
+ f"Unsloth: decision rows with {name} need a newer unsloth-zoo. "
+ "Upgrade with `pip install -U unsloth-zoo`."
+ )
+ return zoo.clef_training_item(pipeline, state, questions, max_length, **media)
def _clef_items(pipeline, rows, tokenizer, max_len, validate, report, skip) -> list:
@@ -432,8 +469,8 @@ def _clef_items(pipeline, rows, tokenizer, max_len, validate, report, skip) -> l
return _decision_clef_items(
rows,
functools.partial(zoo.clef_option_keys, pipeline),
- lambda state, questions, images: _clef_item(
- zoo, pipeline, state, questions, max_len, images
+ lambda state, questions, images, audio: _clef_item(
+ zoo, pipeline, state, questions, max_len, images, audio
),
validate,
report,
@@ -843,9 +880,11 @@ class FastDecisionModel:
state,
questions: dict,
images = None,
+ audio = None,
) -> dict:
"""Answers one Decision API request: {name: answer} at the calibrated temperatures.
`images` (as in a row: PIL images, data URLs, or bytes / path dicts) are read by a Clef whose base model has a vision tower.
+ `audio` (as in a row: data URLs, bytes / path dicts, or decoded 16 kHz mono clips) is heard by a Clef whose base model has an audio tower.
Each answer is the Decision API's (choice / confidence, score / legend, or noul) plus
"answer" (the option, True / False for noul, the level number for score) and
"probabilities" over every option."""
@@ -856,19 +895,19 @@ class FastDecisionModel:
if not isinstance(questions, dict) or not questions:
raise DecisionDataError("questions must be a non-empty dict of name to question")
state, config, zoo = _parsed(state), model.decision_config, _decision_zoo()
- images = _row_images({"images": images})
+ images, audio = _row_images({"images": images}), _row_audio({"audio": audio})
if _is_clef(model):
pipeline = model._unsloth_pipeline
questions = {str(name): _clef_question(q) for name, q in questions.items()}
# Read up to CLEF_SERVE_MAX_LEN tokens, like serving, even past the training cut.
max_length = max(int(config.get("max_len", CLEF_MAX_LEN)), CLEF_SERVE_MAX_LEN)
- item = _clef_item(zoo, pipeline, state, questions, max_length, images)
+ item = _clef_item(zoo, pipeline, state, questions, max_length, images, audio)
logits = zoo.clef_logits(model, [item])[0]
keys = [zoo.clef_option_keys(pipeline, q) for q in questions.values()]
kinds = [QUESTION_TYPES.index(q["type"]) for q in questions.values()]
else:
- if images:
- raise DecisionDataError("Laya reads text only; images need a Clef model.")
+ if images or audio:
+ raise DecisionDataError("Laya reads text only; images and audio need a Clef model.")
common, max_len = _laya().common, int(config.get("max_len", 512))
tokenizer = getattr(tokenizer, "tokenizer", tokenizer)
items, keys = [], []