Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: MLX training from a saved adapter, Continued Pretraining, streamed and audio datasets, with exact resume (#16622)

Long Yixing8611f686fc10
diff --git a/studio/backend/core/training/eval_dataset.py b/studio/backend/core/training/eval_dataset.py
index 13778f22f6c..e54f48c60ac 100644
--- a/studio/backend/core/training/eval_dataset.py
+++ b/studio/backend/core/training/eval_dataset.py
@@ -10,6 +10,8 @@ from typing import Any, Optional
EVAL_SPLIT_CANDIDATES = ("eval", "validation", "valid", "val", "test")
MIN_EVAL_ROWS = 16
MIN_TOTAL_ROWS_FOR_EVAL = MIN_EVAL_ROWS * 2
+# Streaming eval has no __len__, so cap it or an unbounded eval iterates the whole source on every eval step.
+STREAMING_EVAL_MAX_SAMPLES = 500
def evaluation_enabled(value: Any) -> bool:
diff --git a/studio/backend/core/training/provenance.py b/studio/backend/core/training/provenance.py
index 7161970eb06..541608e3c45 100644
--- a/studio/backend/core/training/provenance.py
+++ b/studio/backend/core/training/provenance.py
@@ -24,6 +24,7 @@ _MODEL_LOAD_PREQUANTIZED_4BIT = "prequantized_4bit"
_MODEL_LOAD_RUNTIME_4BIT = "runtime_4bit"
_PREQUANTIZED_MODE_RE = re.compile(r"prequantized_(?:1[0-6]|[1-9])(?:_(?:1[0-6]|[1-9])){0,7}bit")
_REASON_RE = re.compile(r"[a-z0-9][a-z0-9_-]{0,63}")
+_COMMIT_SHA_RE = re.compile(r"[0-9a-f]{40}")
# unsloth/Qwen3.5-* shards are named ``model.safetensors-00001-of-00001.safetensors`` (#15405).
_MODEL_WEIGHT_RE = re.compile(
r"(?:"
@@ -93,6 +94,14 @@ class ExactResumeResourcesUnavailable(ValueError):
pass
+def resume_requires_original_load_mode(config: dict[str, Any]) -> bool:
+ return bool(
+ config.get("require_exact_resume_resources")
+ or config.get("require_exact_model_resource")
+ or config.get("resume_expected_model")
+ )
+
+
def effective_training_load_in_4bit(
config: dict[str, Any], model_load_target: str, hf_token: Optional[str]
) -> bool:
@@ -101,9 +110,7 @@ def effective_training_load_in_4bit(
from utils.transformers_version import latest_tier_active_for
latest_tier_active = latest_tier_active_for(model_load_target, hf_token)
- if latest_tier_active and (
- config.get("require_exact_resume_resources") or config.get("require_exact_model_resource")
- ):
+ if latest_tier_active and resume_requires_original_load_mode(config):
raise ExactResumeResourcesUnavailable(
"This checkpoint requires its original 4-bit model load mode, "
"which is unavailable with the active Transformers runtime."
@@ -121,8 +128,12 @@ def exact_resume_requires_current_4bit(config: dict[str, Any]) -> bool:
return False
except Exception:
return False
- # Same disjunction effective_training_load_in_4bit tests.
- return bool(requires_exact_model or resource_provenance_is_complete(config))
+ # The stored-run form of resume_requires_original_load_mode.
+ return bool(
+ requires_exact_model
+ or resource_provenance_is_complete(config)
+ or local_model_resume_expectation(config) is not None
+ )
def _normalized_repo_id(value: Any) -> Optional[str]:
@@ -642,6 +653,16 @@ def attest_loaded_model(
return None, None, None, reason
+def streamed_dataset_revision(config: dict[str, Any]) -> Optional[str]:
+ """The Hub commit a streaming run is pinned to, or None when it streams a moving branch."""
+ if not config.get("dataset_streaming"):
+ return None
+ revision = config.get("dataset_revision")
+ if isinstance(revision, str) and _COMMIT_SHA_RE.fullmatch(revision):
+ return revision
+ return None
+
+
def _dataset_reason(config: dict[str, Any]) -> str:
if config.get("dataset_streaming"):
return "dataset_streaming_unattested"
@@ -671,12 +692,14 @@ def build_worker_provenance_event(
dataset_snapshot = None
dataset_reason = None
+ # A stream has no cache snapshot; the commit both loaders streamed from attests it.
+ dataset_revision = streamed_dataset_revision(config)
if dataset_loaded_from_exact_snapshot:
dataset_snapshot = exact_dataset_snapshot_path(
config.get("dataset_snapshot_path"),
config.get("hf_dataset"),
)
- if dataset_snapshot is None:
+ if dataset_snapshot is None and dataset_revision is None:
dataset_reason = _dataset_reason(config)
reasons = [reason for reason in (model_reason, dataset_reason) if reason]
@@ -690,8 +713,9 @@ def build_worker_provenance_event(
"load_mode": model_load_mode,
},
"dataset": {
- "status": _ATTESTED if dataset_snapshot else _INCOMPLETE,
+ "status": _ATTESTED if dataset_snapshot or dataset_revision else _INCOMPLETE,
"snapshot_path": dataset_snapshot,
+ "revision": dataset_revision,
},
"reasons": reasons,
}
@@ -769,6 +793,7 @@ def normalize_worker_provenance_event(
reasons.append("model_event_invalid")
dataset_snapshot = None
+ dataset_attested = False
if (
event.get("version") == RESOURCE_PROVENANCE_VERSION
and dataset_event.get("status") == _ATTESTED
@@ -777,10 +802,15 @@ def normalize_worker_provenance_event(
dataset_event.get("snapshot_path"),
config.get("hf_dataset"),
)
- if dataset_snapshot is None and "dataset_event_invalid" not in reasons:
+ # A stream attests only the commit this process pinned, never one the worker names.
+ pinned = streamed_dataset_revision(config)
+ dataset_attested = dataset_snapshot is not None or (
+ pinned is not None and dataset_event.get("revision") == pinned
+ )
+ if not dataset_attested and "dataset_event_invalid" not in reasons:
reasons.append("dataset_event_invalid")
- complete = model_snapshot is not None and dataset_snapshot is not None
+ complete = model_snapshot is not None and dataset_attested
return {
"actual_model_repo_id": model_repo_id,
"model_snapshot_path": model_snapshot,
@@ -790,7 +820,7 @@ def normalize_worker_provenance_event(
"status": "complete" if complete else "incomplete",
"model_status": _ATTESTED if model_snapshot else _INCOMPLETE,
"model_load_mode": model_load_mode,
- "dataset_status": _ATTESTED if dataset_snapshot else _INCOMPLETE,
+ "dataset_status": _ATTESTED if dataset_attested else _INCOMPLETE,
"reasons": reasons,
},
}
@@ -854,7 +884,14 @@ def validate_exact_model_pin(config: dict[str, Any]) -> str:
return model_snapshot
-def validate_exact_dataset_pin(config: dict[str, Any]) -> str:
+def validate_exact_dataset_pin(config: dict[str, Any]) -> Optional[str]:
+ """The dataset snapshot to pin, or None for a stream, which is pinned by its commit instead."""
+ if config.get("dataset_streaming"):
+ if streamed_dataset_revision(config) is None:
+ raise ExactResumeResourcesUnavailable(
+ "The dataset revision used by this run was not attested."
+ )
+ return None
dataset_snapshot = exact_dataset_snapshot_path(
config.get("dataset_snapshot_path"),
config.get("hf_dataset"),
@@ -866,7 +903,7 @@ def validate_exact_dataset_pin(config: dict[str, Any]) -> str:
return dataset_snapshot
-def validate_exact_resource_pins(config: dict[str, Any]) -> tuple[str, str]:
+def validate_exact_resource_pins(config: dict[str, Any]) -> tuple[str, Optional[str]]:
model_snapshot = validate_exact_model_pin(config)
dataset_snapshot = validate_exact_dataset_pin(config)
return model_snapshot, dataset_snapshot
@@ -924,6 +961,52 @@ def exact_resume_resource_requirements(config: dict[str, Any]) -> tuple[bool, bo
return require_model, require_dataset
+def local_model_resume_expectation(config: dict[str, Any]) -> Optional[dict[str, Any]]:
+ """The attested base of a stored run whose model is a local folder (a continued adapter).
+ Resume loads that folder as the run did, so its base is checked once loaded instead of
+ pinned as the load target, which would load the bare base."""
+ from utils.paths import is_local_path
+
+ if not is_local_path(str(config.get("model_name") or "")):
+ return None
+ repo_id = _normalized_repo_id(config.get("actual_model_repo_id"))
+ snapshot = config.get("model_snapshot_path")
+ if repo_id is None or not isinstance(snapshot, str) or not snapshot:
+ # A resume that stopped before attesting hands on the expectation it was checked against.
+ carried = config.get("resume_expected_model")
+ if isinstance(carried, dict) and carried.get("repo_id") and carried.get("snapshot_path"):
+ return carried
+ return None
+ marker = config.get(RESOURCE_PROVENANCE_KEY)
+ return {
+ "repo_id": repo_id,
+ "snapshot_path": snapshot,
+ "load_mode": marker.get("model_load_mode") if isinstance(marker, dict) else None,
+ }
+
+
+def resumed_model_mismatch(expected: dict[str, Any], event: dict[str, Any]) -> Optional[str]:
+ """Why the model a resumed run loaded is not the one its checkpoint was trained on, or None."""
+ model = event.get("model") if isinstance(event.get("model"), dict) else {}
+ loaded = model.get("snapshot_path")
+ if (
+ model.get("status") != _ATTESTED
+ or _normalized_repo_id(model.get("repo_id")) != expected.get("repo_id")
+ or not isinstance(loaded, str)
+ or os.path.realpath(loaded) != os.path.realpath(expected.get("snapshot_path") or "")
+ ):
+ return (
+ f"The base model {expected.get('repo_id')} no longer resolves to the revision this "
+ "checkpoint was trained on, so it cannot be resumed exactly."
+ )
+ if expected.get("load_mode") and model.get("load_mode") != expected["load_mode"]:
+ return (
+ f"The base model loaded as {model.get('load_mode')!r}, but this checkpoint was "
+ f"trained on a {expected['load_mode']!r} load, so it cannot be resumed exactly."
+ )
+ return None
+
+
def resource_provenance_allows_resume(config: dict[str, Any]) -> bool:
return resource_provenance_resume_blocker(config) is None
diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py
index bda6658aaee..fa41cffe76e 100644
--- a/studio/backend/core/training/trainer.py
+++ b/studio/backend/core/training/trainer.py
@@ -53,8 +53,15 @@ import time
from pathlib import Path
from typing import Any, Dict, List, Optional, Callable, Union
from datasets import Dataset
-from core.training.eval_dataset import evaluation_enabled
+from core.training.eval_dataset import STREAMING_EVAL_MAX_SAMPLES, evaluation_enabled
from utils.datasets.audio_decode import ensure_audio_decoding
+from utils.datasets.audio_rows import (
+ audio_transcription_messages,
+ image_slot_count,
+ prepare_audio_vlm_image_column,
+ resolve_audio_columns,
+ resolve_audio_vlm_image_column,
+)
from utils.datasets.tokenizer_fingerprint import install_tokenizer_fingerprint
from utils.datasets.cache_safe import load_dataset_cache_safe as load_dataset
from utils.hf_dataset_options import hf_dataset_split_instruction_names
@@ -124,9 +131,6 @@ _STUDIO_DEFAULT_TARGETS = (
)
_CPT_EMBEDDING_TARGETS = ("embed_tokens", "lm_head")
-# Streaming eval has no __len__, so cap it or an unbounded eval iterates the whole source on every eval step.
-STREAMING_EVAL_MAX_SAMPLES = 500
-
def _build_report_targets(training_args) -> list[str] | str:
report_to: list[str] = []
@@ -1160,59 +1164,6 @@ class UnslothTrainer:
f"{len(removed_modules)} modules\n"
)
- def _resolve_audio_columns(
- self,
- dataset,
- custom_format_mapping: dict = None,
- ):
- """Resolve audio/text/speaker columns from user mapping or fallback. Returns dict with keys
- audio_col, text_col, speaker_col (may be None)."""
- cols = dataset.column_names
-
- if custom_format_mapping:
- audio_col = None
- text_col = None
- speaker_col = None
- for col, role in custom_format_mapping.items():
- if role == "audio":
- audio_col = col
- elif role == "text":
- text_col = col
- elif role == "speaker_id":
- speaker_col = col
- if audio_col and audio_col in cols and text_col and text_col in cols:
- return {
- "audio_col": audio_col,
- "text_col": text_col,
- "speaker_col": speaker_col,
- }
-
- audio_col = next((c for c in cols if c.lower() in ("audio", "speech")), None)
- text_col = next(
- (c for c in cols if c.lower() in ("text", "sentence", "transcript", "transcription")),
- None,
- )
-
- speaker_col = None
- if "source" in cols:
- speaker_col = "source"
- elif "speaker_id" in cols:
- speaker_col = "speaker_id"
-
- if audio_col is None or text_col is None or speaker_col is None:
- from hub.utils.dataset_format import detect_multimodal_dataset
-
- detected = detect_multimodal_dataset(dataset)
- audio_col = audio_col or detected["detected_audio_column"]
- text_col = text_col or detected["detected_text_column"]
- speaker_col = speaker_col or detected["detected_speaker_column"]
-
- return {
- "audio_col": audio_col,
- "text_col": text_col,
- "speaker_col": speaker_col,
- }
-
def load_model(
self,
model_name: str,
@@ -2245,7 +2196,7 @@ class UnslothTrainer:
# A leaked pad_to_multiple_of is rejected by EncodecFeatureExtractor.
processor.tokenizer.init_kwargs.pop("pad_to_multiple_of", None)
- resolved = self._resolve_audio_columns(dataset, custom_format_mapping)
+ resolved = resolve_audio_columns(dataset, custom_format_mapping)
audio_col = resolved["audio_col"]
text_col = resolved["text_col"]
speaker_key = resolved["speaker_col"]
@@ -2342,95 +2293,7 @@ class UnslothTrainer:
)
return result_dataset
- @staticmethod
- def _resolve_audio_vlm_image_column(
- dataset,
- custom_format_mapping,
- audio_col,
- text_col = None,
- speaker_col = None,
- ):
- cols = dataset.column_names
- reserved = {audio_col, text_col}
- reserved.update(
- col for col, role in (custom_format_mapping or {}).items() if role and role != "image"
- )
- for col, role in (custom_format_mapping or {}).items():
- if role == "image" and col in cols and col not in reserved:
- return col
- from hub.utils.dataset_format import (
- _first_row,
- _is_image_list_item,
- _is_image_value,
- detect_multimodal_dataset,
- )
-
- from datasets import Image
-
- def image_typed(col):
- feature = (getattr(dataset, "features", None) or {}).get(col)
- inner = feature[0] if isinstance(feature, list) and feature else feature
- return isinstance(getattr(inner, "feature", inner), Image)
-
- if speaker_col and not image_typed(speaker_col):
- reserved.add(speaker_col)
-
- def image_list(value):
- if not isinstance(value, list):
- return False
- items = [
- item
- for item in value
- if item is not None and not (isinstance(item, str) and not item)
- ]
- return bool(items) and all(map(_is_image_list_item, items))
-
- sample = _first_row(dataset) or {}
-
- def empty(value):
- # Not `in (None, "", [])`: that compares a pixel ndarray elementwise and raises.
- return (
- value is None
- or (isinstance(value, str) and not value)
- or (
- isinstance(value, list)
- and all(item is None or (isinstance(item, str) and not item) for item in value)
- )
- )
-
- ordered = list(
- dict.fromkeys(detect_multimodal_dataset(dataset)["multimodal_columns"] + cols)
- )
- ordered = [col for col in ordered if col not in reserved]
- typed = {col: image_typed(col) for col in ordered}
- values = {col: sample.get(col) for col in ordered}
- first_candidate = next(
- (
- i
- for i, col in enumerate(ordered)
- if typed[col] or _is_image_value(values[col]) or image_list(values[col])
- ),
- len(ordered),
- )
- unresolved = {
- col for col in ordered[:first_candidate] if not typed[col] and empty(values[col])
- }
- if unresolved:
- for i, row in enumerate(dataset):
- if i >= 100 or not unresolved:
- break
- for col in tuple(unresolved):
- if not empty(row.get(col)):
- values[col] = row[col]
- unresolved.remove(col)
- return next(
- (
- col
- for col in ordered
- if typed[col] or _is_image_value(values[col]) or image_list(values[col])
- ),
- None,
- )
+ _resolve_audio_vlm_image_column = staticmethod(resolve_audio_vlm_image_column)
def _format_audio_vlm_dataset(
self,
@@ -2439,7 +2302,7 @@ class UnslothTrainer:
):
from datasets import Audio
- resolved = self._resolve_audio_columns(dataset, custom_format_mapping)
+ resolved = resolve_audio_columns(dataset, custom_format_mapping)
audio_col = resolved["audio_col"]
text_col = resolved["text_col"]
if not audio_col or not text_col:
@@ -2455,89 +2318,20 @@ class UnslothTrainer:
dataset = dataset.cast_column(audio_col, Audio(sampling_rate = 16000))
if image_col:
- from datasets import Image, Sequence
-
- feature = dataset.features.get(image_col)
- inner = feature[0] if isinstance(feature, list) else getattr(feature, "feature", None)
-
- pixel_feature, dimensions = feature, 0
- while getattr(pixel_feature, "feature", None) is not None:
- pixel_feature = pixel_feature.feature
- dimensions += 1
- if dimensions <= 1 and getattr(pixel_feature, "dtype", "") in (
- "string",
- "large_string",
- ):
-
- def normalize_paths(values):
- return {
- image_col: [
- [None if item == "" else item for item in value]
- if isinstance(value, list)
- else None
- if value == ""
- else value
- for value in values
- ]
- }
-
- dataset = dataset.map(normalize_paths, batched = True, input_columns = image_col)
- if dimensions in (2, 3) and getattr(pixel_feature, "dtype", "").startswith(
- ("int", "uint", "float")
- ):
- from itertools import islice
-
- import numpy as np
-
- value = next(
- (row[image_col] for row in islice(dataset, 100) if row[image_col] is not None),
- None,
- )
- shape = np.asarray(value).shape if value is not None else ()
- if value is None or len(shape) == 2 or (len(shape) == 3 and shape[-1] in (1, 3, 4)):
- inner = None
-
- # Image(decode = False) yields {"path", "bytes"} dicts the processor rejects.
- def decodes(f):
- return isinstance(f, Image) and f.decode
-
- if inner is not None:
- if not decodes(inner):
- dataset = dataset.cast_column(image_col, Sequence(Image()))
- elif not decodes(feature):
- dataset = dataset.cast_column(image_col, Image())
+ dataset = prepare_audio_vlm_image_column(dataset, image_col)
logger.info(f"Audio VLM dataset also has images in '{image_col}'; training on both\n")
def format_messages(samples):
formatted = {"messages": []}
for idx in range(len(samples[audio_col])):
- audio = samples[audio_col][idx]["array"]
- label = str(samples[text_col][idx])
- user_content = [
- {"type": "audio", "audio": audio},
- {"type": "text", "text": "Please transcribe this audio."},
- ]
- if image_col:
- row_images = samples[image_col][idx]
- if isinstance(row_images, list):
- n_images = sum(1 for i in row_images if i is not None)
- else:
- n_images = 0 if row_images is None else 1
- user_content[:0] = [{"type": "image"} for _ in range(n_images)]
- message = [
- {
- "role": "system",
- "content": [
- {
- "type": "text",
- "text": "You are an assistant that transcribes speech accurately.",
- }
- ],
- },
- {"role": "user", "content": user_content},
- {"role": "assistant", "content": [{"type": "text", "text": label}]},
- ]
- formatted["messages"].append(message)
+ row_images = samples[image_col][idx] if image_col else None
+ formatted["messages"].append(
+ audio_transcription_messages(
+ samples[audio_col][idx]["array"],
+ str(samples[text_col][idx]),
+ image_slot_count(row_images),
+ )
+ )
return formatted
self._update_progress(status_message = "Formatting audio VLM dataset...")
@@ -2576,7 +2370,7 @@ class UnslothTrainer:
END_OF_TEXT = 128009
AUDIO_OFFSET = 128266
- resolved = self._resolve_audio_columns(dataset, custom_format_mapping)
+ resolved = resolve_audio_columns(dataset, custom_format_mapping)
audio_col = resolved["audio_col"]
text_col = resolved["text_col"]
speaker_col = resolved["speaker_col"]
@@ -2756,7 +2550,7 @@ class UnslothTrainer:
spark_code_dir,
).audio_volume_normalize
- resolved = self._resolve_audio_columns(dataset, custom_format_mapping)
+ resolved = resolve_audio_columns(dataset, custom_format_mapping)
audio_col = resolved["audio_col"]
text_col = resolved["text_col"]
speaker_col = resolved["speaker_col"]
@@ -2956,7 +2750,7 @@ class UnslothTrainer:
outetts_code_dir,
).text_normalizations
- resolved = self._resolve_audio_columns(dataset, custom_format_mapping)
+ resolved = resolve_audio_columns(dataset, custom_format_mapping)
audio_col = resolved["audio_col"]
text_col = resolved["text_col"]
if not audio_col or not text_col:
@@ -3189,7 +2983,7 @@ class UnslothTrainer:
WHISPER_SAMPLE_RATE = 16000
- resolved = self._resolve_audio_columns(dataset, custom_format_mapping)
+ resolved = resolve_audio_columns(dataset, custom_format_mapping)
audio_col = resolved["audio_col"]
text_col = resolved["text_col"]
if not audio_col or not text_col:
diff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.py
index 125085f2c04..8ddb78ba7bc 100644
--- a/studio/backend/core/training/training.py
+++ b/studio/backend/core/training/training.py
@@ -271,6 +271,7 @@ def _build_training_worker_config(values: dict[str, Any]) -> dict[str, Any]:
"model_revision": values.get("model_revision"),
"actual_model_repo_id": values.get("actual_model_repo_id"),
"resume_model_load_mode": values.get("resume_model_load_mode"),
+ "resume_expected_model": values.get("resume_expected_model"),
"dataset_known_cached": values.get("dataset_known_cached", False),
"dataset_local_path": values.get("dataset_local_path"),
"dataset_snapshot_path": values.get("dataset_snapshot_path"),
@@ -594,9 +595,16 @@ def _apply_cache_pins(config: dict[str, Any]) -> None:
hf_dataset = config.get("hf_dataset") or ""
requested_ds_pin = config.get("dataset_snapshot_path")
+ requested_revision = config.get("dataset_revision")
ds_claimed = bool(config.get("dataset_known_cached") or config.get("dataset_local_path"))
config["dataset_revision"] = None
- if not hf_dataset or config.get("dataset_streaming"):
+ if hf_dataset and config.get("dataset_streaming"):
+ config["dataset_snapshot_path"] = None
+ # A resume replays the commit the route took from its run; a new run pins the current one.
+ config["dataset_revision"] = (
+ requested_revision if resume else _hub_dataset_commit(config, hf_dataset, warnings)
+ )
+ elif not hf_dataset:
config["dataset_snapshot_path"] = None
elif resume and requested_ds_pin:
from hub.utils.dataset_cache import (
@@ -649,6 +657,27 @@ def _apply_cache_pins(config: dict[str, Any]) -> None:
config["cache_pin_warnings"] = warnings
+def _hub_dataset_commit(
+ config: dict[str, Any], hf_dataset: str, warnings: list[str]
+) -> Optional[str]:
+ """The commit a new streaming run reads, so a resume can replay the same rows."""
+ from huggingface_hub import HfApi
+
+ token = hf_token_arg(
+ config.get("hf_token") or "", allow_ambient_token = config.get("allow_ambient", True)
+ )
+ try:
+ commit = HfApi().dataset_info(hf_dataset, token = token, timeout = 15).sha
+ except Exception as exc:
+ logger.info("Could not resolve the commit of streamed dataset %s: %s", hf_dataset, exc)
+ commit = None
+ if not commit:
+ warnings.append(
+ f"Streaming {hf_dataset} from its latest revision, so this run cannot be resumed."
+ )
+ return commit
+
+
def _finite_or_none(value: Any) -> Optional[float]:
try:
value = float(value)
@@ -965,13 +994,6 @@ class _MLXTrainerAdapter:
if not self._dataset_config:
self._update_progress(error = "Dataset not loaded")
return False
- if self.is_cpt:
- self._update_progress(
- error = "Continued Pretraining is not supported for MLX training yet.",
- is_training = False,
- is_completed = False,
- )
- return False
config = self._build_worker_config(training_args)
event_queue = queue.Queue()
@@ -1855,10 +1877,9 @@ class TrainingBackend:
"retry when it completes."
)
- if (
- config.get("require_exact_resume_resources")
- or config.get("require_exact_model_resource")
- ) and config.get("load_in_4bit"):
+ from .provenance import resume_requires_original_load_mode
+
+ if resume_requires_original_load_mode(config) and config.get("load_in_4bit"):
from .provenance import effective_training_load_in_4bit
effective_training_load_in_4bit(
config,
diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py
index ce56cf08684..c88a21c8e15 100644
--- a/studio/backend/core/training/worker.py
+++ b/studio/backend/core/training/worker.py
@@ -127,11 +127,6 @@ def _dataset_local_files_only(config: dict) -> bool:
return bool(config.get("dataset_snapshot_path"))
-def _uses_mlx_training_backend() -> bool:
- from core.training.training import should_use_mlx_training_backend
- return should_use_mlx_training_backend()
-
-
def _adapter_trains_embeddings(model) -> bool:
"""Whether a loaded adapter already trains embed_tokens or lm_head in full (#13140)."""
for adapter in (getattr(model, "peft_config", None) or {}).values():
@@ -172,11 +167,9 @@ def _untrainable_model_format_error(config: dict) -> str | None:
return "GGUF models are inference-only and cannot be trained."
if model_format == "adapter":
# LoRA / CPT may continue from a saved adapter; full finetune still cannot (#13140).
- # The embedding, decision and MLX runners wrap a fresh adapter; only the CUDA trainer continues one.
+ # The embedding and decision runners wrap a fresh adapter, so they still need a base.
if config.get("training_type") in ("LoRA/QLoRA", "Continued Pretraining") and not (
- config.get("is_embedding", False)
- or config.get("is_decision", False)
- or _uses_mlx_training_backend()
+ config.get("is_embedding", False) or config.get("is_decision", False)
):
from utils.paths import is_local_path
@@ -611,6 +604,7 @@ def _load_hf_train_and_eval_datasets(
from core.training.eval_dataset import (
EVAL_SPLIT_CANDIDATES,
MIN_EVAL_ROWS,
+ STREAMING_EVAL_MAX_SAMPLES,
evaluation_enabled,
)
@@ -633,14 +627,25 @@ def _load_hf_train_and_eval_datasets(
else:
logger.warning(message)
- def load_remote(split: str):
+ def load_remote(split: str, streaming: bool = False):
kwargs = {"split": split, "token": token}
if subset:
kwargs["name"] = subset
if revision:
kwargs["revision"] = revision
+ if streaming:
+ kwargs["streaming"] = True
return load_dataset(hf_dataset, **kwargs)
+ if config.get("dataset_streaming"):
+ # As on CUDA: eval needs its own split, capped so each pass ends.
+ status_callback(f"Streaming dataset: {hf_dataset}")
+ dataset = load_remote(train_split, streaming = True)
+ eval_dataset = None
+ if eval_enabled and eval_split and eval_split != train_split:
+ eval_dataset = load_remote(eval_split, streaming = True).take(STREAMING_EVAL_MAX_SAMPLES)
+ return dataset, eval_dataset
+
if _dataset_local_files_only(config):
status_callback(f"Loading cached dataset: {hf_dataset}")
try:
@@ -940,6 +945,58 @@ def _check_finetune_targets_after_detect(trainer, config: dict) -> None:
# On CPT, MLX trains these regardless of the layer-family flags.
_CPT_TARGET_NAMES = frozenset({"embed_tokens", "lm_head"})
+_MLX_DEFAULT_LORA_TARGETS = (
+ "q_proj",
+ "k_proj",
+ "v_proj",
+ "o_proj",
+ "gate_proj",
+ "up_proj",
+ "down_proj",
+)
+
+
+def _trains_embedding_modules(trainable_names, full_module_keys = ()) -> bool:
+ """Whether the embedding or output head trains in full, from a fresh CPT wrap or a continued adapter.
+
+ ``full_module_keys`` are the weights zoo resolved for embed_tokens / lm_head, which cover heads named otherwise.
+ """
+ for name in trainable_names:
+ if name in full_module_keys:
+ return True
+ parts = set(name.split("."))
+ if parts & _CPT_TARGET_NAMES and not parts & {"lora_a", "lora_b"}:
+ return True
+ return False
+
+
+def _mlx_quantized_cpt_modules(model, names) -> set:
+ """The requested embed_tokens / lm_head this load quantized (a 4-bit checkpoint or load), which zoo cannot
+ train in full. A tied head is the embedding, so it goes with it."""
+ quantized, found = set(), set()
+ for path, module in model.named_modules():
+ leaf = path.rsplit(".", 1)[-1]
+ if leaf in names:
+ found.add(leaf)
+ if any(hasattr(child, "scales") for _, child in module.named_modules()):
+ quantized.add(leaf)
+ if "embed_tokens" in quantized and "lm_head" in names and "lm_head" not in found:
+ quantized.add("lm_head")
+ return quantized
+
+
+def _mlx_cpt_embedding_learning_rate(requested, lr_value, *, trains_embeddings):
+ """The CUDA CPT rule: lr/10 unless set, and dropped when no embedding module trains."""
+ if not trains_embeddings:
+ if requested is not None:
+ logger.warning(
+ "CPT: embedding_learning_rate was provided but neither embed_tokens "
+ "nor lm_head is being trained; ignoring the override."
+ )
+ return None
+ if requested is None:
+ return lr_value / 10.0
+ return float(requested)
def _names_a_cpt_target(target_modules) -> bool:
@@ -2497,6 +2554,66 @@ def _resize_mlx_vlm_images(
return _resize_mlx_vlm_image(value, resize, image_layout = image_layout)
+_RESOLVE_IMAGE_COLUMN = object()
+
+
+def _mlx_audio_rows(
+ dataset,
+ processor,
+ custom_format_mapping = None,
+ image_col = _RESOLVE_IMAGE_COLUMN,
+) -> tuple:
+ """The CUDA transcription rows, with each clip left decoded so zoo can check its rate, and
+ the row's images (a mixed audio + image dataset) beside them. Returns (rows, image column);
+ an eval split takes the train split's image column, None for none."""
+ from datasets import Audio
+ from unsloth_zoo.mlx.utils import audio_extractor_sampling_rate
+
+ from utils.datasets.audio_decode import ensure_audio_decoding
+ from utils.datasets.audio_rows import (
+ audio_transcription_messages,
+ prepare_audio_vlm_image_column,
+ resolve_audio_columns,
+ resolve_audio_vlm_image_column,
+ )
+
+ if not ensure_audio_decoding():
+ raise RuntimeError(
+ "No audio decoder is available: install the audio runtime (librosa and soundfile)."
+ )
+ columns = resolve_audio_columns(dataset, custom_format_mapping)
+ audio_col, text_col = columns["audio_col"], columns["text_col"]
+ if not audio_col or not text_col:
+ raise ValueError(
+ f"Audio dataset needs 'audio' and 'text' columns, got: {dataset.column_names}"
+ )
+ if image_col is _RESOLVE_IMAGE_COLUMN:
+ image_col = resolve_audio_vlm_image_column(
+ dataset, custom_format_mapping, audio_col, text_col, columns["speaker_col"]
+ )
+ elif image_col not in dataset.column_names:
+ image_col = None
+ rate = audio_extractor_sampling_rate(processor) or 16000
+ dataset = dataset.cast_column(audio_col, Audio(sampling_rate = rate))
+ if image_col:
+ dataset = prepare_audio_vlm_image_column(dataset, image_col)
+ rows = []
+ for row in dataset:
+ images = row[image_col] if image_col else None
+ if not isinstance(images, list):
+ images = [] if images is None else [images]
+ images = [image for image in images if image is not None]
+ item = {
+ "messages": audio_transcription_messages(
+ row[audio_col], str(row[text_col]), len(images)
+ )
+ }
+ if images:
+ item["images"] = images
+ rows.append(item)
+ return rows, image_col
+
+
def _adapt_for_mlx_vlm(
items,
resize = None,
@@ -2767,6 +2884,15 @@ def _resolve_mlx_training_steps(
)
+def _slice_streaming_dataset(dataset, slice_start, slice_end):
+ """A stream slices lazily, as on CUDA, and has no length to bound."""
+ if slice_start:
+ dataset = dataset.skip(slice_start)
+ if slice_end is not None:
+ dataset = dataset.take(max(0, slice_end - (slice_start or 0) + 1))
+ return dataset
+
+
def _configure_mlx_training_schedule(
trainer,
requested_max_steps,
@@ -3025,6 +3151,7 @@ def _run_mlx_training(event_queue, stop_queue, config):
_send("status", status_message = "Loading MLX libraries...")
import mlx.core as mx
+ from mlx.utils import tree_flatten
try:
from unsloth_zoo.mlx.loader import FastMLXModel
@@ -3033,6 +3160,7 @@ def _run_mlx_training(event_queue, stop_queue, config):
MLXTrainingConfig,
train_on_responses_only,
)
+ from unsloth_zoo.mlx.utils import iter_mlx_lora_modules
except ImportError as e:
raise ImportError(
"Unsloth: MLX training requires unsloth-zoo with the MLX modules "
@@ -3066,10 +3194,6 @@ def _run_mlx_training(event_queue, stop_queue, config):
message = "Embedding model training is not supported for MLX training yet."
_send("error", error = message)
raise NotImplementedError(message)
- if config.get("training_type") == "Continued Pretraining":
- message = "Continued Pretraining is not supported for MLX training yet."
- _send("error", error = message)
- raise NotImplementedError(message)
# Decided before the model loads, so version skew does not cost a download.
try:
mlx_dora_kwargs = _mlx_dora_peft_kwargs(config, FastMLXModel.get_peft_model)
@@ -3083,11 +3207,18 @@ def _run_mlx_training(event_queue, stop_queue, config):
# Force text-only for non-image datasets even on vision-capable models (e.g. Qwen3.5-VL on alpaca).
# Forward resume_from_checkpoint to MLXTrainer.train(); else Resume silently restarts from step 0.
resume_from_checkpoint = config.get("resume_from_checkpoint") or None
- is_dataset_image = bool(config.get("is_dataset_image", False))
training_type = config.get("training_type", "LoRA/QLoRA")
- use_lora = training_type == "LoRA/QLoRA"
+ is_cpt = training_type == "Continued Pretraining"
+ # As on CUDA, CPT trains on raw text, so an image dataset loads and formats as text.
+ is_dataset_image = bool(config.get("is_dataset_image", False)) and not is_cpt
+ # CUDA takes its raw-text path (CPT or format "raw") before the audio one.
+ is_dataset_audio = bool(config.get("is_dataset_audio", False)) and not (
+ is_cpt or config.get("format_type") == "raw"
+ )
+ use_lora = training_type in ("LoRA/QLoRA", "Continued Pretraining")
# Before the download/load below: unlike the CUDA path, this needs none of the model.
- if use_lora:
+ # A continued adapter keeps its own targets, and CPT builds its adapter from target_modules alone.
+ if use_lora and not is_cpt and config.get("model_format") != "adapter":
_check_mlx_finetune_targets(config)
# Normalize seed; explicit None must not reach the seed chain.
_raw_seed = config.get("random_seed", 3407)
@@ -3105,7 +3236,12 @@ def _run_mlx_training(event_queue, stop_queue, config):
# Fetch the dataset before the model so a dataset error does not wait on the model download.
_send("status", status_message = "Loading dataset...")
+ if config.get("is_dataset_audio"):
+ # Raw-text runs still read rows carrying the Audio column, so install the decoder for them too.
+ from utils.datasets.audio_decode import ensure_audio_decoding
+ ensure_audio_decoding()
hf_dataset = config.get("hf_dataset", "")
+ dataset_streaming = bool(hf_dataset and config.get("dataset_streaming"))
config["_dataset_loaded_from_exact_snapshot"] = False
def _load_local(file_paths):
@@ -3176,7 +3312,7 @@ def _run_mlx_training(event_queue, stop_queue, config):
model_load_name,
load_in_4bit = config.get("load_in_4bit", True),
full_finetuning = not use_lora,
- text_only = None if is_dataset_image else True,
+ text_only = None if is_dataset_image or is_dataset_audio else True,
token = hf_token,
trust_remote_code = bool(config.get("trust_remote_code", False)),
random_state = model_random_state,
@@ -3211,7 +3347,7 @@ def _run_mlx_training(event_queue, stop_queue, config):
model_load_name,
load_in_4bit = config.get("load_in_4bit", True),
full_finetuning = not use_lora,
- text_only = None if is_dataset_image else True,
+ text_only = None if is_dataset_image or is_dataset_audio else True,
token = hf_token,
trust_remote_code = bool(config.get("trust_remote_code", False)),
random_state = model_random_state,
@@ -3232,8 +3368,15 @@ def _run_mlx_training(event_queue, stop_queue, config):
)
loaded_model_for_provenance = model
- is_vlm = bool(is_dataset_image and getattr(model, "_is_vlm_model", False))
+ is_vlm = bool((is_dataset_image or is_dataset_audio) and getattr(model, "_is_vlm_model", False))
model._is_vlm_model = is_vlm
+ if is_dataset_audio and not is_vlm:
+ # zoo trains audio only as input to an audio-capable vision-language model.
+ raise ValueError(
+ "Audio dataset training on Apple Silicon needs an audio-input model such as "
+ "Gemma 3n or Gemma 4; text-to-speech and speech-recognition checkpoints are not "
+ "supported on MLX yet."
+ )
vision_image_size = config.get("vision_image_size")
# DeepSeek OCR uses a coupled preset tuple; skip resize like the Torch path.
_model_name_lower = str(config.get("model_name", "")).lower()
@@ -3261,7 +3404,42 @@ def _run_mlx_training(event_queue, stop_queue, config):
else:
use_grad_checkpoint = gc_setting
- if use_lora:
+ # A reloaded adapter stays trainable; never stack a second one on it (#13140).
+ continues_adapter = use_lora and next(iter_mlx_lora_modules(model), None) is not None
+ if continues_adapter:
+ _send(
+ "status",
+ status_message = "Continuing from existing LoRA adapters (kept the adapter's settings)",
+ )
+ elif is_cpt:
+ _send("status", status_message = "Configuring LoRA for continued pretraining...")
+ # As on CUDA, embed_tokens / lm_head train as full modules rather than through LoRA.
+ user_modules = config.get("target_modules") or []
+ leaf = lambda module: str(module).rsplit(".", 1)[-1] # noqa: E731
+ full_modules = {leaf(m) for m in user_modules} & _CPT_TARGET_NAMES
+ quantized = _mlx_quantized_cpt_modules(model, full_modules)
+ if quantized:
+ _send(
+ "warning",
+ message = (
+ f"This 4-bit load quantized {' and '.join(sorted(quantized))}, which cannot "
+ "train in full, so continued pretraining leaves them frozen. Load the 16-bit "
+ "model to train them."
+ ),
+ )
+ full_modules -= quantized
+ peft_kwargs = dict(
+ r = config.get("lora_r", 128),
+ lora_alpha = config.get("lora_alpha", 32),
+ lora_dropout = config.get("lora_dropout", 0.0),
+ use_rslora = config.get("use_rslora", False),
+ random_state = lora_random_state,
+ target_modules = [m for m in user_modules if leaf(m) not in _CPT_TARGET_NAMES]
+ or list(_MLX_DEFAULT_LORA_TARGETS),
+ modules_to_save = sorted(full_modules) or None,
+ use_gradient_checkpointing = use_grad_checkpoint,
+ )
+ elif use_lora:
_send("status", status_message = "Configuring LoRA adapters...")
peft_kwargs = dict(
r = config.get("lora_r", 16),
@@ -3270,23 +3448,16 @@ def _run_mlx_training(event_queue, stop_queue, config):
use_rslora = config.get("use_rslora", False),
init_lora_weights = config.get("init_lora_weights", True),
random_state = lora_random_state,
- target_modules = config.get("target_modules")
- or [
- "q_proj",
- "k_proj",
- "v_proj",
- "o_proj",
- "gate_proj",
- "up_proj",
- "down_proj",
- ],
+ target_modules = config.get("target_modules") or list(_MLX_DEFAULT_LORA_TARGETS),
use_gradient_checkpointing = use_grad_checkpoint,
)
- peft_kwargs.update(mlx_dora_kwargs)
finetune_language = config.get("finetune_language_layers", True)
finetune_attention = config.get("finetune_attention_modules", True)
finetune_mlp = config.get("finetune_mlp_modules", True)
- finetune_vision = config.get("finetune_vision_layers", False) if is_vlm else False
+ # Audio-only rows give vision LoRA no gradient, and zoo refuses Gemma 3n's linear-free vision tower.
+ finetune_vision = (
+ config.get("finetune_vision_layers", False) if is_vlm and is_dataset_image else False
+ )
if (finetune_attention or finetune_mlp) and not finetune_language and not finetune_vision:
finetune_language = True
@@ -3303,6 +3474,8 @@ def _run_mlx_training(event_queue, stop_queue, config):
peft_kwargs["finetune_mlp_modules"] = finetune_mlp
if is_vlm:
peft_kwargs["finetune_vision_layers"] = finetune_vision
+ if use_lora and not continues_adapter:
+ peft_kwargs.update(mlx_dora_kwargs)
model = FastMLXModel.get_peft_model(model, **peft_kwargs)
slice_start = config.get("dataset_slice_start")
@@ -3333,6 +3506,8 @@ def _run_mlx_training(event_queue, stop_queue, config):
_send("status", status_message = f"Using {kept} of {total} rows (max_steps run)")
def _slice(ds):
+ if dataset_streaming:
+ return _slice_streaming_dataset(ds, slice_start, slice_end)
if slice_start is not None or slice_end is not None:
start = slice_start if slice_start is not None else 0
end = slice_end if slice_end is not None else len(ds) - 1
@@ -3351,17 +3526,18 @@ def _run_mlx_training(event_queue, stop_queue, config):
dataset = _slice(dataset)
- _emit_resource_provenance(
+ if not _emit_resource_provenance(
event_queue,
config,
loaded_model_for_provenance,
model_load_target = model_load_name,
model_load_in_4bit = bool(config.get("load_in_4bit")),
dataset_loaded_from_exact_snapshot = bool(config.get("_dataset_loaded_from_exact_snapshot")),
- )
+ ):
+ return
# Reuse the GPU format pipeline for VLM (OCR/caption/llava/sharegpt+images) and text.
- format_type = config.get("format_type", "")
+ format_type = "raw" if is_cpt else config.get("format_type", "")
custom_format_mapping = config.get("custom_format_mapping")
dataset_final_format = ""
eval_final_format = ""
@@ -3370,7 +3546,27 @@ def _run_mlx_training(event_queue, stop_queue, config):
def _fmt_progress(status_message = "", **_kw):
_send("status", status_message = status_message)
- if is_vlm:
+ if is_dataset_audio:
+ _send("status", status_message = "Formatting audio dataset...")
+ dataset, audio_image_col = _mlx_audio_rows(dataset, tokenizer, custom_format_mapping)
+ if eval_dataset is not None:
+ eval_dataset, _ = _mlx_audio_rows(
+ eval_dataset, tokenizer, custom_format_mapping, audio_image_col
+ )
+ if audio_image_col:
+ image_layout = (
+ _mlx_vlm_resized_image_layout(tokenizer)
+ if vision_image_size is not None
+ else None
+ )
+ dataset = _adapt_for_mlx_vlm(
+ dataset, resize = vision_image_size, image_layout = image_layout
+ )
+ if eval_dataset is not None:
+ eval_dataset = _adapt_for_mlx_vlm(
+ eval_dataset, resize = vision_image_size, image_layout = image_layout
+ )
+ elif is_vlm:
_send("status", status_message = "Formatting VLM dataset...")
vlm_info = format_and_template_dataset(
dataset,
@@ -3522,7 +3718,19 @@ def _run_mlx_training(event_queue, stop_queue, config):
batch_size = config.get("batch_size", 4)
grad_accum = config.get("gradient_accumulation_steps", 4)
- lr_value = float(config.get("learning_rate", "2e-4"))
+ lr_value = float(config.get("learning_rate", "5e-5" if is_cpt else "2e-4"))
+ embedding_lr_value = (
+ _mlx_cpt_embedding_learning_rate(
+ config.get("embedding_learning_rate"),
+ lr_value,
+ trains_embeddings = _trains_embedding_modules(
+ (name for name, _ in tree_flatten(model.trainable_parameters())),
+ getattr(model, "_unsloth_cpt_full_module_weight_keys", None) or (),
+ ),
+ )
+ if is_cpt
+ else None
+ )
# Warmup: prefer warmup_steps; fall back to warmup_ratio
warmup_steps = config.get("warmup_steps")
@@ -3570,7 +3778,7 @@ def _run_mlx_training(event_queue, stop_queue, config):
weight_decay = config.get("weight_decay", 0.001)
weight_decay = 0.001 if weight_decay is None else float(weight_decay)
- # `streaming` stays off: without a pass length zoo cannot end an epoch on an optimizer step or report a real epoch.
+ # A sized dataset stays eager: without a pass length zoo cannot end an epoch on an optimizer step.
mlx_config_kwargs = dict(
per_device_train_batch_size = batch_size,
gradient_accumulation_steps = grad_accum,
@@ -3589,6 +3797,7 @@ def _run_mlx_training(event_queue, stop_queue, config):
compile = True,
gradient_checkpointing = use_grad_checkpoint,
packing = bool(config.get("packing", False)),
+ streaming = dataset_streaming,
output_dir = output_dir,
save_steps = int(config.get("save_steps", 0) or 0),
eval_steps = eval_steps_val,
@@ -3606,7 +3815,8 @@ def _run_mlx_training(event_queue, stop_queue, config):
True if _raw_cast is None else bool(_raw_cast)
)
if "dataset_order" in _supported_fields:
- mlx_config_kwargs["dataset_order"] = "torch_randperm"
+ # CUDA's trainer reads a stream in source order; a sized dataset gets its RandomSampler order.
+ mlx_config_kwargs["dataset_order"] = "sequential" if dataset_streaming else "torch_randperm"
if "max_grad_leaf_norm" in _supported_fields:
mlx_config_kwargs["max_grad_leaf_norm"] = max_grad_leaf_norm
if "report_grad_norm" in _supported_fields:
@@ -3623,6 +3833,8 @@ def _run_mlx_training(event_queue, stop_queue, config):
if patience and eval_dataset is not None and eval_steps_value > 0:
mlx_config_kwargs["early_stopping_patience"] = patience
mlx_config_kwargs["load_best_model_at_end"] = True
+ if embedding_lr_value is not None:
+ mlx_config_kwargs["embedding_learning_rate"] = embedding_lr_value
if "append_eos" in _supported_fields:
# Unsloth SFT formatting owns rendered examples; raw/CPT text still needs MLX to append EOS.
mlx_config_kwargs["append_eos"] = bool(raw_text_mode)
@@ -3637,7 +3849,7 @@ def _run_mlx_training(event_queue, stop_queue, config):
max_steps, eval_steps_val = _configure_mlx_training_schedule(
trainer,
requested_max_steps,
- len(dataset),
+ None if dataset_streaming else len(dataset),
batch_size,
grad_accum,
num_epochs,
@@ -3895,17 +4107,6 @@ def run_mlx_training_process(
)
return
- if config.get("is_dataset_audio"):
- event_queue.put(
- {
- "type": "error",
- "error": "Audio dataset training is not yet supported on Apple Silicon.",
- "stack": "",
- "ts": time.time(),
- }
- )
- return
-
try:
try:
if config.get("is_decision"):
@@ -5156,7 +5357,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
)
return
- _emit_resource_provenance(
+ if not _emit_resource_provenance(
event_queue,
config,
trainer.model,
@@ -5165,7 +5366,8 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
dataset_loaded_from_exact_snapshot = bool(
getattr(trainer, "dataset_loaded_from_exact_snapshot", False)
),
- )
+ ):
+ return
if eval_dataset is not None:
event_queue.put(
@@ -5454,10 +5656,12 @@ def _emit_resource_provenance(
model_load_target: str,
model_load_in_4bit: bool,
dataset_loaded_from_exact_snapshot: bool,
-) -> None:
+) -> bool:
+ """False, after reporting why, when a resumed run loaded a different base than it trained on."""
from core.training.provenance import (
build_worker_provenance_event,
incomplete_worker_provenance_event,
+ resumed_model_mismatch,
)
try:
@@ -5472,7 +5676,15 @@ def _emit_resource_provenance(
logger.warning("Could not attest training resource provenance", exc_info = True)
event = incomplete_worker_provenance_event("provenance_attestation_failed")
event["ts"] = time.time()
+ expected = config.get("resume_expected_model")
+ mismatch = resumed_model_mismatch(expected, event) if expected else None
+ if mismatch:
+ # Record only the dataset, so the next resume still checks this run's expected base.
+ event_queue.put({**event, "model": incomplete_worker_provenance_event()["model"]})
+ event_queue.put({"type": "error", "error": mismatch, "stack": "", "ts": time.time()})
+ return False
event_queue.put(event)
+ return True
def _mlx_has_checkpoint_at_step(output_dir, step: int) -> bool:
@@ -6017,14 +6229,15 @@ def _run_embedding_training(event_queue: Any, stop_queue: Any, config: dict) ->
event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()})
return
- _emit_resource_provenance(
+ if not _emit_resource_provenance(
event_queue,
config,
loaded_model_for_provenance,
model_load_target = model_load_name,
model_load_in_4bit = False,
dataset_loaded_from_exact_snapshot = bool(config.get("_dataset_loaded_from_exact_snapshot")),
- )
+ ):
+ return
loss = MultipleNegativesRankingLoss(model)
diff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.py
index 6b83de37f50..a0113020602 100644
--- a/studio/backend/routes/training.py
+++ b/studio/backend/routes/training.py
@@ -472,23 +472,28 @@ def _has_adapter_metadata(path: Path) -> bool:
def _continue_from_adapter_allowed(request: TrainingStartRequest) -> bool:
"""LoRA / CPT can load a saved adapter as the starting point; full finetune cannot (#13140).
- Only the CUDA trainer keeps an adapter it loaded; the embedding, decision and Apple Silicon
- (MLX) runners always build a fresh one on top, so those still need a base model."""
+ The embedding and decision runners always build a fresh adapter on top, so those still need
+ a base model."""
if request.training_type not in ("LoRA/QLoRA", "Continued Pretraining"):
return False
- if getattr(request, "is_embedding", False) or getattr(request, "is_decision", False):
- return False
- from core.training.training import should_use_mlx_training_backend
-
- return not should_use_mlx_training_backend()
+ return not (getattr(request, "is_embedding", False) or getattr(request, "is_decision", False))
# The names PEFT and Unsloth's loader open; numbered adapter shards are not loaded.
_ADAPTER_WEIGHT_FILES = ("adapter_model.safetensors", "adapter_model.bin")
+# The MLX loader imports PEFT safetensors (not .bin) and reads its own saved format.
+_MLX_ADAPTER_WEIGHT_FILES = ("adapter_model.safetensors", "adapters.safetensors")
def _has_local_adapter_weights(path: Path) -> bool:
- return path.is_dir() and any((path / name).is_file() for name in _ADAPTER_WEIGHT_FILES)
+ if not path.is_dir():
+ return False
+ from core.training.training import should_use_mlx_training_backend
+
+ names = (
+ _MLX_ADAPTER_WEIGHT_FILES if should_use_mlx_training_backend() else _ADAPTER_WEIGHT_FILES
+ )
+ return any((path / name).is_file() for name in names)
_HF_MODEL_ACCESS_DENIED = (
@@ -1340,21 +1345,11 @@ def _validate_training_platform(request: TrainingStartRequest) -> None:
)
if hardware.DEVICE != hardware.DeviceType.MLX:
return
- if request.training_type == "Continued Pretraining":
- raise HTTPException(
- status_code = 400,
- detail = "Continued Pretraining is not supported for MLX training yet.",
- )
if request.is_embedding:
raise HTTPException(
status_code = 400,
detail = "Embedding model training is not supported for MLX training yet.",
)
- if request.is_dataset_audio:
- raise HTTPException(
- status_code = 400,
- detail = "Audio dataset training is not yet supported on Apple Silicon.",
- )
if request.use_loftq:
raise HTTPException(
status_code = 400,
@@ -1549,13 +1544,15 @@ def _normalized_optional_string(value: Any) -> Optional[str]:
def _prepare_resume_resource_provenance(
request: TrainingStartRequest, resume_run: dict
-) -> tuple[Optional[str], bool, bool, bool, Optional[str]]:
+) -> tuple[Optional[str], bool, bool, bool, Optional[str], dict[str, Any]]:
stored = training_run_config(resume_run)
from core.training.provenance import (
ExactResumeResourcesUnavailable,
RESOURCE_PROVENANCE_KEY,
exact_resume_resource_requirements,
+ local_model_resume_expectation,
resource_provenance_is_complete,
+ streamed_dataset_revision,
)
try:
@@ -1628,6 +1625,22 @@ def _prepare_resume_resource_provenance(
) from error
for field in TrainingStartRequest.model_fields:
setattr(request, field, getattr(validated_request, field))
+ overrides: dict[str, Any] = {}
+ stream_revision = streamed_dataset_revision(stored)
+ if stream_revision is not None:
+ # Replay the streamed commit even if the run never attested, not the moving branch.
+ overrides["dataset_revision"] = stream_revision
+ expected_model = local_model_resume_expectation(stored)
+ if expected_model is not None:
+ # The worker checks the base it loads against this; the dataset pin still applies.
+ return (
+ None,
+ False,
+ requires_exact_dataset,
+ False,
+ None,
+ {**overrides, "resume_expected_model": expected_model},
+ )
marker = stored.get(RESOURCE_PROVENANCE_KEY)
model_load_mode = (
_normalized_optional_string(marker.get("model_load_mode"))
@@ -1640,6 +1653,7 @@ def _prepare_resume_resource_provenance(
requires_exact_dataset,
resource_provenance_is_complete(stored),
model_load_mode,
+ overrides,
)
@@ -1961,6 +1975,7 @@ async def _admit_training_request(
resume_requires_exact_resources = False
resume_requires_exact_model = False
resume_requires_exact_dataset = False
+ resume_overrides: dict[str, Any] = {}
if request.resume_from_checkpoint:
try:
resume_output_dir = await asyncio.to_thread(
@@ -2010,6 +2025,7 @@ async def _admit_training_request(
resume_requires_exact_dataset,
resume_requires_exact_resources,
resume_model_load_mode,
+ resume_overrides,
) = await asyncio.to_thread(
_prepare_resume_resource_provenance,
request,
@@ -2048,11 +2064,6 @@ async def _admit_training_request(
status_code = 400,
detail = "dataset_streaming is not supported for embedding training; the embedding loader needs the full dataset.",
)
- if _hw.DEVICE == _hw.DeviceType.MLX:
- raise HTTPException(
- status_code = 400,
- detail = "dataset_streaming is not yet supported on Apple Silicon (MLX); the MLX loader materializes the full dataset.",
- )
if request.max_steps is None or request.max_steps <= 0:
raise HTTPException(
status_code = 422,
@@ -2246,6 +2257,7 @@ async def _admit_training_request(
"gpu_ids": request.gpu_ids,
"s3_config": request.s3_config.model_dump() if request.s3_config else None,
}
+ training_kwargs.update(resume_overrides)
# Latest-sidecar models size and train 16-bit (same flip as chat load): 4-bit is disabled for brand-new
# architectures, so VRAM checks must not underestimate a load the worker refuses.
diff --git a/studio/backend/tests/test_audio_detected_columns.py b/studio/backend/tests/test_audio_detected_columns.py
index 89ae55046d2..d3c1b4bf715 100644
--- a/studio/backend/tests/test_audio_detected_columns.py
+++ b/studio/backend/tests/test_audio_detected_columns.py
@@ -12,57 +12,16 @@ _BACKEND = Path(__file__).resolve().parent.parent
if str(_BACKEND) not in sys.path:
sys.path.insert(0, str(_BACKEND))
-import importlib # noqa: E402
-import types # noqa: E402
-from unittest.mock import MagicMock # noqa: E402
-
import pytest # noqa: E402
from datasets import Dataset # noqa: E402
from hub.utils.dataset_format import check_dataset_format # noqa: E402
-
-
-_STUBBED: list[str] = []
-
-
-def _stub_if_missing(name, attrs):
- if name in sys.modules:
- return
- try:
- importlib.import_module(name)
- return
- except Exception: # noqa: BLE001 - stub unusable imports
- pass
- _STUBBED.append(name)
- mod = types.ModuleType(name)
- mod.__spec__ = None
- for attr in attrs:
- setattr(mod, attr, MagicMock())
- sys.modules[name] = mod
- parent, _, child = name.rpartition(".")
- if parent and parent in sys.modules:
- setattr(sys.modules[parent], child, mod)
-
-
-_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))
-_stub_if_missing("unsloth.chat_templates", ("get_chat_template",))
-_stub_if_missing("trl", ("SFTTrainer", "SFTConfig"))
-
-from core.training import trainer as tmod # noqa: E402
-
-for _name in reversed(_STUBBED):
- sys.modules.pop(_name, None)
+from utils.datasets.audio_rows import resolve_audio_columns # noqa: E402
_AUDIO = {"array": [0.0] * 160, "sampling_rate": 16000}
-def audio_trainer(monkeypatch):
- monkeypatch.setattr(tmod, "should_use_mlx_training_backend", lambda *a, **k: False)
- return tmod.UnslothTrainer()
-
-
@pytest.mark.parametrize(
"row, expected",
[
@@ -74,38 +33,38 @@ def audio_trainer(monkeypatch):
({"audio": _AUDIO, "text": "hi", "speaker": "a"}, ("audio", "text", "speaker")),
],
)
-def test_resolves_the_columns_the_check_accepted(audio_trainer, row, expected):
+def test_resolves_the_columns_the_check_accepted(row, expected):
dataset = Dataset.from_list([row])
check = check_dataset_format(dataset, is_vlm = False)
assert check["requires_manual_mapping"] is False
- resolved = audio_trainer._resolve_audio_columns(dataset, None)
+ resolved = resolve_audio_columns(dataset, None)
assert (resolved["audio_col"], resolved["text_col"], resolved["speaker_col"]) == expected
assert resolved["audio_col"] == check["detected_audio_column"]
-def test_exact_names_and_explicit_mapping_are_unchanged(audio_trainer):
+def test_exact_names_and_explicit_mapping_are_unchanged():
dataset = Dataset.from_list(
[{"audio": _AUDIO, "text": "hi", "prompt": "p", "speaker": "a", "source": "0"}]
)
- assert audio_trainer._resolve_audio_columns(dataset, None) == {
+ assert resolve_audio_columns(dataset, None) == {
"audio_col": "audio",
"text_col": "text",
"speaker_col": "source",
}
- assert audio_trainer._resolve_audio_columns(dataset, {"audio": "audio", "prompt": "text"}) == {
+ assert resolve_audio_columns(dataset, {"audio": "audio", "prompt": "text"}) == {
"audio_col": "audio",
"text_col": "prompt",
"speaker_col": None,
}
-def test_text_only_dataset_still_resolves_nothing(audio_trainer):
+def test_text_only_dataset_still_resolves_nothing():
dataset = Dataset.from_list([{"instruction": "a", "output": "b"}])
- assert audio_trainer._resolve_audio_columns(dataset, None) == {
+ assert resolve_audio_columns(dataset, None) == {
"audio_col": None,
"text_col": None,
"speaker_col": None,
diff --git a/studio/backend/tests/test_audio_token_detection.py b/studio/backend/tests/test_audio_token_detection.py
index 00f699917ca..ca4c8550cc4 100644
--- a/studio/backend/tests/test_audio_token_detection.py
+++ b/studio/backend/tests/test_audio_token_detection.py
@@ -683,3 +683,139 @@ def test_lfm3_audio_on_the_hub_is_audio_vlm(monkeypatch):
model = "LiquidAI/LFM3-Audio-2B",
)
assert (audio_type, definitive) == ("audio_vlm", True)
+
+
+# Gemma 4's transformers 5 tokenizer_config: no added_tokens_decoder, a named audio_token. Every size
+# carries it; only E2B/E4B/12B have an audio encoder.
+_V5_AUDIO_TOKENIZER = {"audio_token": "<|audio|>", "image_token": "<|image|>"}
+_WITH_ENCODER = {"model_type": "gemma4", "audio_config": {"model_type": "gemma4_audio"}}
+_WITHOUT_ENCODER = {"model_type": "gemma4", "audio_config": None}
+
+
+def _escaped(tok_config):
+ return json.dumps(tok_config).replace("<", chr(92) + "u003c").replace(">", chr(92) + "u003e")
+
+
+ "tok_config, model_config, expected",
+ [
+ (_V5_AUDIO_TOKENIZER, _WITH_ENCODER, ("audio_vlm", True)),
+ (_V5_AUDIO_TOKENIZER, _WITHOUT_ENCODER, (None, True)),
+ (_V5_AUDIO_TOKENIZER, None, (None, False)),
+ ({"image_token": "<|image|>"}, _WITH_ENCODER, (None, True)),
+ (
+ {**_V5_AUDIO_TOKENIZER, "added_tokens_decoder": {"0": {"content": "<|bicodec_x|>"}}},
+ _WITH_ENCODER,
+ ("bicodec", True),
+ ),
+ ],
+)
+def test_a_declared_audio_token_needs_an_audio_encoder(tok_config, model_config, expected):
+ from utils.models.model_config import _classify_audio_tokenizer
+ assert _classify_audio_tokenizer(tok_config, lambda: model_config) == expected
+
+
+ "tokenizer_text", [json.dumps(_V5_AUDIO_TOKENIZER), _escaped(_V5_AUDIO_TOKENIZER)]
+)
+ "model_config, expected", [(_WITH_ENCODER, "audio_vlm"), (_WITHOUT_ENCODER, None)]
+)
+def test_a_cached_gemma4_snapshot_is_typed_by_its_audio_encoder(
+ monkeypatch, tmp_path, tokenizer_text, model_config, expected
+):
+ repo_dir, _ = _cached_snapshot(
+ tmp_path,
+ "acme/tts-model",
+ {"tokenizer_config.json": tokenizer_text, "config.json": json.dumps(model_config)},
+ )
+
+ (audio_type, definitive), reads, _documents = _detect_against_cache(monkeypatch, repo_dir)
+
+ assert (audio_type, definitive, reads) == (expected, True, [])
+
+
+ "tokenizer_text", [json.dumps(_V5_AUDIO_TOKENIZER), _escaped(_V5_AUDIO_TOKENIZER)]
+)
[email protected]("config_text", ["{half", None])
+def test_a_cached_config_it_cannot_read_leaves_the_hub_to_answer(
+ monkeypatch, tmp_path, tokenizer_text, config_text
+):
+ # None: a tokenizer-only download, whose snapshot never fetched config.json.
+ files = {"tokenizer_config.json": tokenizer_text}
+ if config_text is not None:
+ files["config.json"] = config_text
+ repo_dir, _ = _cached_snapshot(tmp_path, "acme/tts-model", files)
+
+ (audio_type, _definitive), reads, _documents = _detect_against_cache(
+ monkeypatch,
+ repo_dir,
+ listed = ("tokenizer_config.json", "config.json"),
+ responses = [_Resp(200, _V5_AUDIO_TOKENIZER), _Resp(200, _WITH_ENCODER)],
+ )
+
+ assert audio_type == "audio_vlm"
+ assert reads[-1].endswith("/config.json")
+
+
+def test_a_cached_adapter_repo_answers_without_the_hub(monkeypatch, tmp_path):
+ # An adapter repo never ships config.json, so its absence is not a partial download.
+ repo_dir, _ = _cached_snapshot(
+ tmp_path,
+ "acme/tts-model",
+ {"tokenizer_config.json": json.dumps(_V5_AUDIO_TOKENIZER), "adapter_config.json": "{}"},
+ )
+
+ (audio_type, definitive), reads, _documents = _detect_against_cache(
+ monkeypatch, repo_dir, listed = ("tokenizer_config.json", "adapter_config.json")
+ )
+
+ assert (audio_type, definitive, reads) == (None, True, [])
+
+
+def test_a_local_folder_without_a_config_is_definitively_not_audio(tmp_path):
+ from utils.models import model_config
+ (tmp_path / "tokenizer_config.json").write_text(json.dumps(_V5_AUDIO_TOKENIZER))
+
+ assert model_config._detect_audio_from_tokenizer(str(tmp_path), local_files_only = True) == (
+ None,
+ True,
+ )
+
+
+ "config_response, expected",
+ [
+ (_Resp(200, _WITH_ENCODER), ("audio_vlm", True)),
+ (_Resp(200, _WITHOUT_ENCODER), (None, True)),
+ (_Resp(404), (None, True)),
+ (_Resp(503), (None, False)),
+ ],
+)
+def test_the_hub_config_decides_a_declared_audio_token(monkeypatch, config_response, expected):
+ responses = [_Resp(200, _V5_AUDIO_TOKENIZER), config_response, _Resp(404)]
+ assert _detect_checked(monkeypatch, responses) == expected
+
+
+def test_an_unread_encoder_config_stays_unknown_past_a_readable_processor(monkeypatch):
+ # Gemma 4's processor_config carries no top-level audio_token, so it must not settle the answer.
+ responses = [
+ _Resp(200, _V5_AUDIO_TOKENIZER),
+ _Resp(503),
+ _Resp(404),
+ _Resp(200, {"feature_extractor": {}}),
+ ]
+ assert _detect_checked(monkeypatch, responses) == (None, False)
+
+
+def test_a_local_folder_with_an_escaped_audio_token_reads_its_encoder(tmp_path):
+ from utils.models import model_config
+
+ (tmp_path / "tokenizer_config.json").write_text(_escaped(_V5_AUDIO_TOKENIZER))
+ (tmp_path / "config.json").write_text(json.dumps(_WITH_ENCODER))
+
+ assert model_config._detect_audio_from_tokenizer(str(tmp_path), local_files_only = True) == (
+ "audio_vlm",
+ True,
+ )
diff --git a/studio/backend/tests/test_embedding_csv_cells_as_written.py b/studio/backend/tests/test_embedding_csv_cells_as_written.py
index 6910d3085f1..377942f102e 100644
--- a/studio/backend/tests/test_embedding_csv_cells_as_written.py
+++ b/studio/backend/tests/test_embedding_csv_cells_as_written.py
@@ -100,7 +100,7 @@ def run_embedding(monkeypatch, tmp_path):
)
monkeypatch.setattr(worker, "_model_load_security_error", lambda *a, **k: None)
monkeypatch.setattr(worker, "_start_worker_stop_poller", lambda *a, **k: None)
- monkeypatch.setattr(worker, "_emit_resource_provenance", lambda *a, **k: None)
+ monkeypatch.setattr(worker, "_emit_resource_provenance", lambda *a, **k: True)
def run(
csv_text,
diff --git a/studio/backend/tests/test_export_multi_gpu_device_map.py b/studio/backend/tests/test_export_multi_gpu_device_map.py
index 875a784e8cc..0a65c56aa51 100644
--- a/studio/backend/tests/test_export_multi_gpu_device_map.py
+++ b/studio/backend/tests/test_export_multi_gpu_device_map.py
@@ -10,6 +10,8 @@ import sys
import types
from pathlib import Path
+import pytest
+
_BACKEND_DIR = Path(__file__).resolve().parent.parent
if str(_BACKEND_DIR) not in sys.path:
sys.path.insert(0, str(_BACKEND_DIR))
@@ -556,3 +558,30 @@ def test_the_retry_names_sequential_rather_than_omitting_the_device_map(monkeypa
assert len(_RefusesAnyPlan.calls) == 2
assert _RefusesAnyPlan.calls[0]["device_map"] == "unsloth_balanced"
assert _RefusesAnyPlan.calls[1]["device_map"] == "sequential"
+
+
[email protected]("audio_type", [None, "audio_vlm"])
+def test_an_audio_input_vision_model_exports_through_the_vision_load(
+ monkeypatch, tmp_path, audio_type
+):
+ mod = _export_mod(monkeypatch)
+ _RecordingLoader.calls = []
+ monkeypatch.setattr(mod, "FastVisionModel", _RecordingLoader)
+ monkeypatch.setattr(
+ mod, "FastLanguageModel", lambda *a, **k: pytest.fail("loaded as a text model")
+ )
+ monkeypatch.setattr(mod, "detect_audio_type", lambda *a, **k: audio_type)
+ monkeypatch.setattr(mod, "is_vision_model", lambda *a, **k: True)
+ monkeypatch.setattr(mod, "_hf_offline", lambda *a, **k: False)
+ monkeypatch.setattr(mod, "_offline_window_if", lambda flag: contextlib.nullcontext())
+ monkeypatch.setattr(mod, "_multi_gpu_device_map_kwargs", lambda: {})
+ checkpoint = tmp_path / "checkpoint-100"
+ checkpoint.mkdir()
+ backend = mod.ExportBackend.__new__(mod.ExportBackend)
+ backend.cleanup_memory = lambda: None
+
+ ok, message = backend.load_checkpoint(str(checkpoint))
+
+ assert ok, message
+ assert backend.is_vision is True
+ assert len(_RecordingLoader.calls) == 1
diff --git a/studio/backend/tests/test_mlx_dataset_format_failure.py b/studio/backend/tests/test_mlx_dataset_format_failure.py
index 1eddefda896..fad30f99244 100644
--- a/studio/backend/tests/test_mlx_dataset_format_failure.py
+++ b/studio/backend/tests/test_mlx_dataset_format_failure.py
@@ -149,6 +149,11 @@ def _run_mlx_until_model_load(monkeypatch, config, stop_requested):
mlx_core = types.ModuleType("mlx.core")
mlx_core.metal = types.SimpleNamespace(is_available = lambda: False)
mlx.core = mlx_core
+ mlx_utils = types.ModuleType("mlx.utils")
+ mlx_utils.tree_flatten = lambda tree: []
+ mlx.utils = mlx_utils
+ zoo_utils = types.ModuleType("unsloth_zoo.mlx.utils")
+ zoo_utils.iter_mlx_lora_modules = lambda model: iter(())
loader = types.ModuleType("unsloth_zoo.mlx.loader")
loader.FastMLXModel = _FastMLXModel
trainer = types.ModuleType("unsloth_zoo.mlx.trainer")
@@ -163,6 +168,8 @@ def _run_mlx_until_model_load(monkeypatch, config, stop_requested):
for name, module in {
"mlx": mlx,
"mlx.core": mlx_core,
+ "mlx.utils": mlx_utils,
+ "unsloth_zoo.mlx.utils": zoo_utils,
"unsloth_zoo.mlx.loader": loader,
"unsloth_zoo.mlx.trainer": trainer,
"core.training.s3_dataset": s3_dataset,
diff --git a/studio/backend/tests/test_mlx_epoch_report.py b/studio/backend/tests/test_mlx_epoch_report.py
index 1e61fa3bd43..31e659901a3 100644
--- a/studio/backend/tests/test_mlx_epoch_report.py
+++ b/studio/backend/tests/test_mlx_epoch_report.py
@@ -58,6 +58,7 @@ def _run_step_callback(
"resume_from_checkpoint": None,
"slice_start": None,
"slice_end": None,
+ "dataset_streaming": False,
"mlx_split_names_rows": False,
"mlx_max_train_rows": max_train_rows,
"mlx_max_train_rows_seed": 3407,
@@ -90,18 +91,18 @@ def test_mlx_step_callback_counts_a_bounded_run_over_the_whole_dataset():
assert progress[0]["epoch"] == 0.5
-def test_mlx_training_never_streams():
- keyword_values = [
- n.value for n in ast.walk(_TREE) if isinstance(n, ast.keyword) and n.arg == "streaming"
- ]
- subscript_values = [
- n.value
+def test_mlx_training_streams_only_a_streamed_hf_dataset():
+ """A sized dataset must keep the eager path, or zoo cannot end an epoch on an optimizer step."""
+ assigned = [
+ ast.unparse(n.value)
for n in ast.walk(_TREE)
if isinstance(n, ast.Assign)
- and any(
- isinstance(t, ast.Subscript) and ast.unparse(t.slice) == "'streaming'"
- for t in n.targets
- )
+ and any(isinstance(t, ast.Name) and t.id == "dataset_streaming" for t in n.targets)
+ ]
+ assert assigned == ["bool(hf_dataset and config.get('dataset_streaming'))"]
+ keyword_values = [
+ ast.unparse(n.value)
+ for n in ast.walk(_TREE)
+ if isinstance(n, ast.keyword) and n.arg == "streaming"
]
- for value in keyword_values + subscript_values:
- assert isinstance(value, ast.Constant) and value.value is False, ast.unparse(value)
+ assert keyword_values == ["dataset_streaming"]
diff --git a/studio/backend/tests/test_mlx_training_studio_apple.py b/studio/backend/tests/test_mlx_training_studio_apple.py
new file mode 100644
index 00000000000..0242c8312f9
--- /dev/null
+++ b/studio/backend/tests/test_mlx_training_studio_apple.py
@@ -0,0 +1,128 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Real _run_mlx_training runs (CPT, continuing a saved adapter, streaming), on Apple Silicon only."""
+
+from __future__ import annotations
+
+import json
+import math
+import platform
+import queue
+import sys
+
+import pytest
+
+pytestmark = [
+ pytest.mark.skipif(
+ sys.platform != "darwin" or platform.machine() != "arm64",
+ reason = "MLX runs on Apple Silicon only",
+ ),
+ pytest.mark.allow_network,
+]
+
+_TINY = "mlx-community/SmolLM-135M-Instruct-4bit"
+
+
+class _Events:
+ def __init__(self):
+ self.events = []
+
+ def put(self, event):
+ self.events.append(event)
+
+ def of(self, kind):
+ return [e for e in self.events if e.get("type") == kind]
+
+ def statuses(self):
+ return [e.get("status_message") or "" for e in self.of("status")]
+
+
+def _rows(tmp_path):
+ path = tmp_path / "rows.jsonl"
+ with path.open("w", encoding = "utf-8") as f:
+ for i in range(16):
+ f.write(
+ json.dumps(
+ {
+ "instruction": f"Add {i} and {i}.",
+ "input": "",
+ "output": f"{i} plus {i} is {2 * i}.",
+ "text": f"Row {i}: the number {i} doubled is {2 * i}.",
+ }
+ )
+ + "\n"
+ )
+ return path
+
+
+def _train(tmp_path, name, **overrides):
+ pytest.importorskip("mlx_lm")
+ pytest.importorskip("unsloth_zoo.mlx.loader")
+ import core.training.worker as worker
+
+ config = {
+ "model_name": _TINY,
+ "training_type": "LoRA/QLoRA",
+ "load_in_4bit": True,
+ "max_seq_length": 128,
+ "max_steps": 2,
+ "num_epochs": 1,
+ "batch_size": 1,
+ "gradient_accumulation_steps": 1,
+ "learning_rate": "2e-4",
+ "warmup_steps": 0,
+ "eval_steps": 0,
+ "save_steps": 0,
+ "lora_r": 8,
+ "lora_alpha": 16,
+ "random_seed": 3407,
+ "output_dir": str(tmp_path / name),
+ "allow_external_output_dir": True,
+ "local_datasets": [str(_rows(tmp_path))],
+ **overrides,
+ }
+ events = _Events()
+ worker._run_mlx_training(events, queue.Queue(), config)
+ assert not events.of("error"), events.of("error")
+ complete = events.of("complete")
+ assert complete and complete[-1].get("output_dir"), events.events[-5:]
+ losses = [e.get("loss") for e in events.of("progress") if e.get("loss") is not None]
+ assert losses and all(math.isfinite(loss) for loss in losses), losses
+ return events, complete[-1]["output_dir"]
+
+
+def test_mlx_continued_pretraining_on_a_4bit_checkpoint(tmp_path):
+ events, _ = _train(
+ tmp_path,
+ "cpt",
+ training_type = "Continued Pretraining",
+ target_modules = ["q_proj", "v_proj", "embed_tokens", "lm_head"],
+ )
+ assert any("continued pretraining" in s for s in events.statuses())
+ # The quantized embedding cannot train in full; the run says so instead of failing in zoo.
+ assert any("embed_tokens" in e.get("message", "") for e in events.of("warning"))
+
+
+def test_mlx_lora_continues_from_its_saved_adapter(tmp_path):
+ _, adapter_dir = _train(tmp_path, "first")
+ events, _ = _train(
+ tmp_path,
+ "second",
+ model_name = adapter_dir,
+ model_format = "adapter",
+ )
+ statuses = events.statuses()
+ assert any("Continuing from existing LoRA adapters" in s for s in statuses), statuses
+ assert not any("Configuring LoRA adapters" in s for s in statuses), statuses
+
+
+def test_mlx_streams_a_hub_dataset(tmp_path):
+ events, _ = _train(
+ tmp_path,
+ "stream",
+ local_datasets = None,
+ hf_dataset = "yahma/alpaca-cleaned",
+ dataset_streaming = True,
+ )
+ assert any("Streaming dataset" in s for s in events.statuses())
diff --git a/studio/backend/tests/test_mlx_training_worker_config.py b/studio/backend/tests/test_mlx_training_worker_config.py
index f7b26f4f75a..30fc8512b02 100644
--- a/studio/backend/tests/test_mlx_training_worker_config.py
+++ b/studio/backend/tests/test_mlx_training_worker_config.py
@@ -79,6 +79,7 @@ def _load_worker_module():
"flash_attn_wheel_url",
"install_wheel",
"is_windows_wheel_env",
+ "mamba_ssm_pure_wheel_url",
"pinned_source_build_supported",
"probe_torch_wheel_env",
"url_exists",
@@ -614,3 +615,492 @@ def test_mlx_single_process_finalization_keeps_prior_contract():
events = _finalize(trainer, stop = (True, True), checkpoint_ok = False)
assert trainer.saved == ["/out"]
assert events[-1][0] == "error" and events[-1][1]["resume_blocked"] is True
+
+
+def _adapter_setup_block():
+ """The `continues_adapter = ...` line, the LoRA setup `if` and the wrap after it in _run_mlx_training."""
+ body = ast.parse(textwrap.dedent(inspect.getsource(_worker._run_mlx_training))).body[0].body
+ starts = [
+ i
+ for i, node in enumerate(body)
+ if isinstance(node, ast.Assign) and ast.unparse(node.targets[0]) == "continues_adapter"
+ ]
+ assert len(starts) == 1, "expected one continues_adapter assignment"
+ return body[starts[0] : starts[0] + 3]
+
+
+def _run_adapter_setup(
+ lora_modules,
+ training_type = "LoRA/QLoRA",
+ modules = (),
+ **config,
+):
+ """Lifts the real adapter setup out of _run_mlx_training, which only runs on Apple Silicon."""
+ block = compile(ast.Module(body = _adapter_setup_block(), type_ignores = []), "<peft>", "exec")
+ events, wraps = [], []
+ model = types.SimpleNamespace(lora_modules = lora_modules, named_modules = lambda: modules)
+
+ def get_peft_model(wrapped, **kwargs):
+ wraps.append(kwargs)
+ return wrapped
+
+ namespace = dict(vars(_worker))
+ namespace.update(
+ {
+ "config": config,
+ "model": model,
+ "training_type": training_type,
+ "is_cpt": _run_mlx_assignment("is_cpt", training_type = training_type),
+ "use_lora": _run_mlx_assignment("use_lora", training_type = training_type),
+ "is_vlm": False,
+ "use_grad_checkpoint": "mlx",
+ "lora_random_state": 3407,
+ "mlx_dora_kwargs": {},
+ "FastMLXModel": types.SimpleNamespace(get_peft_model = get_peft_model),
+ "iter_mlx_lora_modules": lambda m: iter(m.lora_modules),
+ "_send": lambda event_type, **kw: events.append((event_type, kw)),
+ }
+ )
+ exec(block, namespace)
+ return events, wraps
+
+
[email protected]("training_type", ["LoRA/QLoRA", "Continued Pretraining"])
+def test_mlx_continued_adapter_is_not_wrapped_again(training_type):
+ events, wraps = _run_adapter_setup(
+ [("layers.0.self_attn.q_proj", object())], training_type = training_type
+ )
+
+ assert wraps == []
+ assert any("Continuing from existing LoRA adapters" in kw["status_message"] for _, kw in events)
+
+
+def test_mlx_base_model_gets_a_fresh_adapter():
+ _, wraps = _run_adapter_setup([], lora_r = 8)
+
+ assert [kwargs["r"] for kwargs in wraps] == [8]
+
+
+def test_mlx_cpt_trains_embedding_modules_in_full():
+ _, wraps = _run_adapter_setup(
+ [],
+ training_type = "Continued Pretraining",
+ target_modules = ["q_proj", "model.embed_tokens", "lm_head"],
+ )
+
+ assert len(wraps) == 1
+ assert wraps[0]["target_modules"] == ["q_proj"]
+ assert wraps[0]["modules_to_save"] == ["embed_tokens", "lm_head"]
+ assert not any(key.startswith("finetune_") for key in wraps[0])
+
+
+def test_mlx_cpt_without_embedding_targets_wraps_the_default_projections():
+ _, wraps = _run_adapter_setup([], training_type = "Continued Pretraining")
+
+ assert wraps[0]["target_modules"] == list(_worker._MLX_DEFAULT_LORA_TARGETS)
+ assert wraps[0]["modules_to_save"] is None
+
+
+def test_trains_embedding_modules_ignores_lora_on_the_head():
+ trains = _worker._trains_embedding_modules
+ assert trains(["model.layers.0.self_attn.q_proj.lora_a", "model.embed_tokens.weight"])
+ assert trains(["model.layers.0.mlp.up_proj.lora_b", "lm_head.weight"])
+ assert not trains(["lm_head.lora_a", "lm_head.lora_b", "model.layers.0.mlp.up_proj.lora_a"])
+
+
+def test_mlx_cpt_embedding_learning_rate_follows_the_cuda_rule(monkeypatch):
+ monkeypatch.setattr(_worker, "logger", types.SimpleNamespace(warning = lambda *_a: None))
+ rate = _worker._mlx_cpt_embedding_learning_rate
+ assert rate(None, 5e-5, trains_embeddings = True) == pytest.approx(5e-6)
+ assert rate(1e-6, 5e-5, trains_embeddings = True) == 1e-6
+ assert rate(1e-6, 5e-5, trains_embeddings = False) is None
+
+
+def _trainer_config_block():
+ """From the MLXTrainingConfig kwargs through the schedule call in _run_mlx_training."""
+ body = ast.parse(textwrap.dedent(inspect.getsource(_worker._run_mlx_training))).body[0].body
+ targets = [
+ ast.unparse(node.targets[0]) if isinstance(node, ast.Assign) else "" for node in body
+ ]
+ start = targets.index("mlx_config_kwargs")
+ end = targets.index("(max_steps, eval_steps_val)")
+ return body[start : end + 1]
+
+
+class _UnsizedStream:
+ def __len__(self):
+ raise TypeError("a stream has no length")
+
+
[email protected]("embedding_lr_value", [5e-6, None])
[email protected]("dataset_streaming", [True, False])
+def test_mlx_trainer_config_follows_streaming_and_embedding_lr(
+ dataset_streaming, embedding_lr_value
+):
+ block = compile(ast.Module(body = _trainer_config_block(), type_ignores = []), "<cfg>", "exec")
+ configs, schedules = [], []
+ namespace = dict(vars(_worker))
+ namespace.update(
+ {
+ "config": {},
+ "dataset": _UnsizedStream() if dataset_streaming else [0] * 7,
+ "dataset_streaming": dataset_streaming,
+ "embedding_lr_value": embedding_lr_value,
+ "training_type": "LoRA/QLoRA",
+ "format_type": "alpaca",
+ "MLXTrainingConfig": type(
+ "MLXTrainingConfig",
+ (),
+ {
+ "__dataclass_fields__": {"dataset_order": None},
+ "__init__": lambda self, **kw: configs.append(kw),
+ },
+ ),
+ "MLXTrainer": lambda **kw: kw,
+ "_configure_mlx_training_schedule": lambda *a, **kw: schedules.append(a) or (8, 0),
+ **{
+ name: None
+ for name in (
+ "model",
+ "tokenizer",
+ "eval_dataset",
+ "eval_steps_value",
+ "max_grad_leaf_norm",
+ "requested_max_steps",
+ "batch_size",
+ "grad_accum",
+ "num_epochs",
+ "warmup_ratio",
+ "warmup_uses_ratio",
+ "eval_steps_ratio",
+ "warmup_steps",
+ "lr_value",
+ "lr_scheduler_type",
+ "optim_name",
+ "weight_decay",
+ "max_grad_norm",
+ "max_grad_value",
+ "max_seq_length",
+ "random_seed",
+ "use_grad_checkpoint",
+ "output_dir",
+ "eval_steps_val",
+ )
+ },
+ }
+ )
+ exec(block, namespace)
+
+ assert configs[0]["streaming"] is dataset_streaming
+ assert configs[0]["dataset_order"] == ("sequential" if dataset_streaming else "torch_randperm")
+ assert configs[0].get("embedding_learning_rate") == embedding_lr_value
+ assert schedules[0][2] == (None if dataset_streaming else 7)
+
+
+def test_mlx_slice_reads_a_stream_lazily():
+ body = ast.parse(textwrap.dedent(inspect.getsource(_worker._run_mlx_training))).body[0].body
+ nodes = [n for n in body if isinstance(n, ast.FunctionDef) and n.name == "_slice"]
+ calls = []
+
+ class Stream(_UnsizedStream):
+ def skip(self, n):
+ calls.append(("skip", n))
+ return self
+
+ def take(self, n):
+ calls.append(("take", n))
+ return self
+
+ namespace = {
+ **vars(_worker),
+ "dataset_streaming": True,
+ "slice_start": 2,
+ "slice_end": 4,
+ "mlx_split_names_rows": False,
+ }
+ exec(compile(ast.Module(body = nodes, type_ignores = []), "<slice>", "exec"), namespace)
+ namespace["_slice"](Stream())
+
+ # Rows 2..4 inclusive.
+ assert calls == [("skip", 2), ("take", 3)]
+
+
+def _run_mlx_assignment(target, **names):
+ body = ast.parse(textwrap.dedent(inspect.getsource(_worker._run_mlx_training))).body[0].body
+ nodes = [n for n in body if isinstance(n, ast.Assign) and ast.unparse(n.targets[0]) == target]
+ assert len(nodes) == 1
+ namespace = {**vars(_worker), **names}
+ exec(compile(ast.Module(body = nodes, type_ignores = []), "<assign>", "exec"), namespace)
+ return namespace[target]
+
+
[email protected]("is_cpt, expected", [(True, "raw"), (False, "alpaca")])
+def test_mlx_cpt_formats_raw_text_whatever_format_was_sent(is_cpt, expected):
+ config = {"format_type": "alpaca"}
+ assert _run_mlx_assignment("format_type", is_cpt = is_cpt, config = config) == expected
+
+
[email protected]("is_cpt", [True, False])
[email protected]("flag", ["is_dataset_image", "is_dataset_audio"])
+def test_mlx_cpt_reads_a_multimodal_dataset_as_text(is_cpt, flag):
+ assert _run_mlx_assignment(flag, is_cpt = is_cpt, config = {flag: True}) is not is_cpt
+
+
[email protected]("format_type", ["raw", "auto"])
+def test_mlx_raw_format_reads_an_audio_dataset_as_text(format_type):
+ config = {"is_dataset_audio": True, "format_type": format_type}
+ is_audio = _run_mlx_assignment("is_dataset_audio", is_cpt = False, config = config)
+ assert is_audio is (format_type != "raw")
+
+
+def test_mlx_cpt_embedding_lr_reaches_heads_zoo_resolved():
+ model = types.SimpleNamespace(
+ trainable_parameters = lambda: {"model.tok_embeddings.weight": None},
+ _unsloth_cpt_full_module_weight_keys = {"model.tok_embeddings.weight"},
+ )
+ rate = _run_mlx_assignment(
+ "embedding_lr_value",
+ is_cpt = True,
+ config = {},
+ lr_value = 5e-5,
+ model = model,
+ tree_flatten = lambda tree: list(tree.items()),
+ )
+ assert rate == pytest.approx(5e-6)
+
+
+def _mlx_body_if(test_source):
+ body = ast.parse(textwrap.dedent(inspect.getsource(_worker._run_mlx_training))).body[0].body
+ nodes = [
+ n
+ for n in ast.walk(ast.Module(body = body, type_ignores = []))
+ if isinstance(n, ast.If) and ast.unparse(n.test) == test_source
+ ]
+ assert len(nodes) == 1
+ return compile(ast.Module(body = nodes, type_ignores = []), "<if>", "exec")
+
+
[email protected]("is_vlm", [True, False])
+def test_mlx_audio_dataset_needs_an_audio_vlm(is_vlm):
+ block = _mlx_body_if("is_dataset_audio and (not is_vlm)")
+ namespace = {**vars(_worker), "is_dataset_audio": True, "is_vlm": is_vlm}
+ if is_vlm:
+ exec(block, namespace)
+ else:
+ with pytest.raises(ValueError, match = "audio-input model"):
+ exec(block, namespace)
+
+
[email protected]("image_col", [None, "image"])
+def test_mlx_audio_dataset_formats_both_splits_as_audio_rows(image_col):
+ formatted, adapted = [], []
+ namespace = {
+ **vars(_worker),
+ "is_dataset_audio": True,
+ "is_vlm": True,
+ "dataset": "train",
+ "eval_dataset": "eval",
+ "tokenizer": "processor",
+ "custom_format_mapping": None,
+ "_send": lambda *_a, **_kw: None,
+ "vision_image_size": None,
+ "_mlx_audio_rows": lambda ds, processor, mapping, *column: formatted.append((ds, *column))
+ or ([ds], image_col),
+ "_adapt_for_mlx_vlm": lambda rows, **_kw: adapted.append(rows) or [*rows, "lifted"],
+ "format_and_template_dataset": lambda *_a, **_kw: pytest.fail(
+ "audio skips the VLM formatter"
+ ),
+ }
+ exec(_mlx_body_if("is_dataset_audio"), namespace)
+
+ # The eval split takes the train split's image column, and only a split with images is lifted.
+ assert formatted == [("train",), ("eval", image_col)]
+ assert adapted == ([] if image_col is None else [["train"], ["eval"]])
+ suffix = [] if image_col is None else ["lifted"]
+ assert namespace["dataset"] == ["train", *suffix]
+ assert namespace["eval_dataset"] == ["eval", *suffix]
+
+
[email protected]("audio", [True, False])
+def test_mlx_audio_request_installs_the_decoder(monkeypatch, audio):
+ # A CPT or raw-format run masks is_dataset_audio but still reads rows with an Audio column.
+ from utils.datasets import audio_decode
+
+ calls = []
+ monkeypatch.setattr(audio_decode, "ensure_audio_decoding", lambda: calls.append(1) or True)
+ exec(_mlx_body_if("config.get('is_dataset_audio')"), {"config": {"is_dataset_audio": audio}})
+
+ assert calls == ([1] if audio else [])
+
+
+class _AudioRowsDataset:
+ def __init__(self, rows):
+ self.rows, self.cast = rows, None
+ self.column_names = list(rows[0])
+ self.features = {}
+
+ def cast_column(self, column, feature):
+ if not hasattr(feature, "sampling_rate"):
+ return self
+ # A real cast returns a new dataset decoding at the target rate.
+ rate = feature.sampling_rate
+ cast = _AudioRowsDataset(
+ [{**row, column: {**row[column], "sampling_rate": rate}} for row in self.rows]
+ )
+ cast.cast = (column, rate)
+ return cast
+
+ def __iter__(self):
+ return iter(self.rows)
+
+
+def test_mlx_audio_rows_keep_the_decoded_clip(monkeypatch):
+ from utils.datasets import audio_decode
+
+ decoders = [True]
+ monkeypatch.setattr(audio_decode, "ensure_audio_decoding", lambda: decoders[0])
+ zoo_utils = types.ModuleType("unsloth_zoo.mlx.utils")
+ zoo_utils.audio_extractor_sampling_rate = lambda processor: {"processor": 24000}.get(processor)
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.utils", zoo_utils)
+ clips = [{"array": [0.1], "sampling_rate": 8000}, {"array": [0.2], "sampling_rate": 8000}]
+ dataset = _AudioRowsDataset([{"audio": clip, "text": t} for clip, t in zip(clips, "ab")])
+
+ rows, image_col = _worker._mlx_audio_rows(dataset, "processor")
+
+ assert image_col is None
+ assert [row["messages"][1]["content"][0]["audio"] for row in rows] == [
+ {"array": [0.1], "sampling_rate": 24000},
+ {"array": [0.2], "sampling_rate": 24000},
+ ]
+ unknown_rate, _ = _worker._mlx_audio_rows(dataset, "unknown")
+ assert unknown_rate[0]["messages"][1]["content"][0]["audio"]["sampling_rate"] == 16000
+ assert [row["messages"][2]["content"][0]["text"] for row in rows] == ["a", "b"]
+ with pytest.raises(ValueError, match = "'audio' and 'text'"):
+ _worker._mlx_audio_rows(_AudioRowsDataset([{"audio": clips[0], "x": 1}]), "processor")
+ decoders[0] = False
+ with pytest.raises(RuntimeError, match = "audio runtime"):
+ _worker._mlx_audio_rows(dataset, "processor")
+
+
+def test_mlx_audio_rows_carry_each_rows_images(monkeypatch):
+ from PIL import Image
+
+ from utils.datasets import audio_decode
+
+ monkeypatch.setattr(audio_decode, "ensure_audio_decoding", lambda: True)
+ zoo_utils = types.ModuleType("unsloth_zoo.mlx.utils")
+ zoo_utils.audio_extractor_sampling_rate = lambda processor: 16000
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.utils", zoo_utils)
+ red, blue = Image.new("RGB", (4, 4), "red"), Image.new("RGB", (4, 4), "blue")
+ clip = {"array": [0.1], "sampling_rate": 16000}
+ dataset = _AudioRowsDataset(
+ [
+ {"audio": clip, "photo": [red, blue], "text": "a"},
+ {"audio": clip, "photo": [None], "text": "b"},
+ ]
+ )
+
+ rows, image_col = _worker._mlx_audio_rows(dataset, "processor")
+
+ assert image_col == "photo"
+ assert rows[0]["images"] == [red, blue] and "images" not in rows[1]
+ kinds = [[part["type"] for part in row["messages"][1]["content"]] for row in rows]
+ assert kinds == [["image", "image", "audio", "text"], ["audio", "text"]]
+ # An eval split whose train split had no images trains on audio alone.
+ eval_rows, eval_image_col = _worker._mlx_audio_rows(dataset, "processor", None, None)
+ assert eval_image_col is None and "images" not in eval_rows[0]
+
+
[email protected]("is_dataset_audio", [True, False])
+def test_mlx_audio_dataset_loads_and_trains_as_a_vlm(is_dataset_audio):
+ names = {"is_dataset_image": False, "is_dataset_audio": is_dataset_audio}
+ model = types.SimpleNamespace(_is_vlm_model = True)
+ assert _run_mlx_assignment("is_vlm", model = model, **names) is is_dataset_audio
+ tree = ast.parse(textwrap.dedent(inspect.getsource(_worker._run_mlx_training)))
+ text_only = {
+ ast.unparse(kw.value)
+ for n in ast.walk(tree)
+ if isinstance(n, ast.Call)
+ for kw in n.keywords
+ if kw.arg == "text_only"
+ }
+ assert len(text_only) == 1
+ assert eval(text_only.pop(), names) is (None if is_dataset_audio else True)
+
+
+ "is_dataset_image, is_dataset_audio, expected",
+ [(True, False, True), (False, True, False), (True, True, True)],
+)
+def test_mlx_vision_layers_train_only_on_image_rows(is_dataset_image, is_dataset_audio, expected):
+ tree = ast.parse(textwrap.dedent(inspect.getsource(_worker._run_mlx_training)))
+ nodes = [
+ n
+ for n in ast.walk(tree)
+ if isinstance(n, ast.Assign) and ast.unparse(n.targets[0]) == "finetune_vision"
+ ]
+ assert len(nodes) == 1
+ namespace = {
+ "config": {"finetune_vision_layers": True},
+ "is_vlm": True,
+ "is_dataset_image": is_dataset_image,
+ "is_dataset_audio": is_dataset_audio,
+ }
+ exec(compile(ast.Module(body = nodes, type_ignores = []), "<vision>", "exec"), namespace)
+ assert namespace["finetune_vision"] is expected
+
+
+class _Module:
+ def __init__(
+ self,
+ quantized = False,
+ children = (),
+ ):
+ if quantized:
+ self.scales = object()
+ self._children = list(children)
+
+ def named_modules(self):
+ yield "", self
+ for name, child in self._children:
+ for sub, module in child.named_modules():
+ yield f"{name}.{sub}" if sub else name, module
+
+
+def test_mlx_cpt_leaves_quantized_embeddings_frozen_with_a_warning():
+ modules = [
+ ("model.embed_tokens", _Module(quantized = True)),
+ ("lm_head", _Module(children = [("linear", _Module(quantized = True))])),
+ ]
+ events, wraps = _run_adapter_setup(
+ [],
+ training_type = "Continued Pretraining",
+ modules = modules,
+ target_modules = ["q_proj", "embed_tokens", "lm_head"],
+ )
+
+ assert wraps[0]["target_modules"] == ["q_proj"]
+ assert wraps[0]["modules_to_save"] is None
+ warnings = [kw["message"] for kind, kw in events if kind == "warning"]
+ assert len(warnings) == 1 and "embed_tokens and lm_head" in warnings[0]
+
+
+def test_mlx_cpt_trains_unquantized_embeddings_in_full():
+ modules = [("model.embed_tokens", _Module()), ("lm_head", _Module())]
+ events, wraps = _run_adapter_setup(
+ [],
+ training_type = "Continued Pretraining",
+ modules = modules,
+ target_modules = ["q_proj", "embed_tokens", "lm_head"],
+ )
+
+ assert wraps[0]["modules_to_save"] == ["embed_tokens", "lm_head"]
+ assert not [kw for kind, kw in events if kind == "warning"]
+
+
+def test_mlx_cpt_drops_a_tied_head_with_its_quantized_embedding():
+ quantized = _Module(quantized = True)
+ assert _worker._mlx_quantized_cpt_modules(
+ types.SimpleNamespace(named_modules = lambda: [("model.embed_tokens", quantized)]),
+ {"embed_tokens", "lm_head"},
+ ) == {"embed_tokens", "lm_head"}
diff --git a/studio/backend/tests/test_snac_code_interleave.py b/studio/backend/tests/test_snac_code_interleave.py
index 864e6875f0c..db4dc625da2 100644
--- a/studio/backend/tests/test_snac_code_interleave.py
+++ b/studio/backend/tests/test_snac_code_interleave.py
@@ -114,9 +114,6 @@ def _run(method, rows, *, stop_after, dtype, events, monkeypatch):
def __init__(self):
self.checks = 0
- def _resolve_audio_columns(self, dataset, custom_format_mapping):
- return {"audio_col": "audio", "text_col": "text", "speaker_col": "speaker"}
-
@property
def should_stop(self):
self.checks += 1
@@ -165,6 +162,11 @@ def _run(method, rows, *, stop_after, dtype, events, monkeypatch):
"Dataset": Dataset,
"gc": types.SimpleNamespace(collect = lambda: None),
"clear_gpu_cache": lambda: None,
+ "resolve_audio_columns": lambda dataset, mapping: {
+ "audio_col": "audio",
+ "text_col": "text",
+ "speaker_col": "speaker",
+ },
}
exec(method, namespace)
owner = Owner()
diff --git a/studio/backend/tests/test_training_cached_start.py b/studio/backend/tests/test_training_cached_start.py
index dab4acbc685..15fb6bc913e 100644
--- a/studio/backend/tests/test_training_cached_start.py
+++ b/studio/backend/tests/test_training_cached_start.py
@@ -132,7 +132,7 @@ def _known_cache_root(monkeypatch, tmp_path):
@pytest.fixture
def _cuda_trainer():
- # Continuation needs the CUDA trainer; an Apple Silicon runner would take the MLX refusal.
+ # On the CUDA trainer only PEFT weight names count as adapter weights.
with patch("core.training.training.should_use_mlx_training_backend", return_value = False):
yield
@@ -519,7 +519,7 @@ def test_untrainable_gate_refuses_cached_hub_adapter_snapshot_for_lora(tmp_path)
assert exc_info.value.detail["code"] == "training_hub_adapter_unsupported"
[email protected]("runner", ["embedding", "decision", "mlx"])
[email protected]("runner", ["embedding", "decision"])
def test_start_refuses_adapter_dir_where_runner_builds_a_fresh_adapter(tmp_path, runner):
route = _load_route_module(f"training_route_adapter_dir_{runner}")
(tmp_path / "adapter_config.json").write_text("{}")
@@ -531,14 +531,45 @@ def test_start_refuses_adapter_dir_where_runner_builds_a_fresh_adapter(tmp_path,
is_decision = runner == "decision",
)
- with patch(
- "core.training.training.should_use_mlx_training_backend",
- return_value = runner == "mlx",
- ):
- exc_info = _shared_setup_2(request, route)
+ exc_info = _shared_setup_2(request, route)
assert exc_info.value.detail["code"] == "training_model_adapter_not_trainable"
[email protected]("weights", ["adapter_model.safetensors", "adapters.safetensors"])
+def test_mlx_start_continues_a_peft_or_mlx_adapter_dir(tmp_path, weights):
+ route = _load_route_module(f"training_route_mlx_adapter_dir_{weights}")
+ (tmp_path / "adapter_config.json").write_text("{}")
+ (tmp_path / "config.json").write_text("{}")
+ (tmp_path / weights).write_bytes(b"x")
+ request = _request(model_name = str(tmp_path), model_format = "adapter")
+
+ with patch("core.training.training.should_use_mlx_training_backend", return_value = True):
+ result = route._reject_untrainable_model_request(request)
+ assert result.model_name == str(tmp_path.resolve())
+
+
+def test_mlx_start_refuses_a_bin_only_peft_adapter_dir(tmp_path):
+ route = _load_route_module("training_route_mlx_bin_adapter_dir")
+ (tmp_path / "adapter_config.json").write_text("{}")
+ (tmp_path / "adapter_model.bin").write_bytes(b"x")
+ request = _request(model_name = str(tmp_path), model_format = "adapter")
+
+ with patch("core.training.training.should_use_mlx_training_backend", return_value = True):
+ exc_info = _shared_setup_2(request, route)
+ assert exc_info.value.detail["code"] == "training_local_model_adapter_weights_missing"
+
+
[email protected]("_cuda_trainer")
+def test_cuda_start_refuses_an_mlx_only_adapter_dir(tmp_path):
+ route = _load_route_module("training_route_cuda_mlx_adapter_dir")
+ (tmp_path / "adapter_config.json").write_text("{}")
+ (tmp_path / "adapters.safetensors").write_bytes(b"x")
+ request = _request(model_name = str(tmp_path), model_format = "adapter")
+
+ exc_info = _shared_setup_2(request, route)
+ assert exc_info.value.detail["code"] == "training_local_model_adapter_weights_missing"
+
+
@pytest.mark.parametrize("refused", [True, False])
@pytest.mark.usefixtures("_cuda_trainer")
def test_start_checks_the_cached_base_a_local_adapter_names(tmp_path, refused):
@@ -1815,9 +1846,7 @@ def test_streaming_rejects_cached_dataset_hints(cache_overrides):
@pytest.mark.parametrize(
("request_overrides", "expected"),
[
- ({"training_type": "Continued Pretraining"}, "Continued Pretraining"),
({"is_embedding": True}, "Embedding model training"),
- ({"is_dataset_audio": True}, "Audio dataset training"),
({"use_loftq": True}, "LoftQ"),
],
)
@@ -1838,7 +1867,7 @@ def test_mlx_start_rejects_unsupported_training_config(request_overrides, expect
assert expected in exc_info.value.detail
-def test_mlx_start_accepts_dora():
+def test_mlx_start_accepts_dora_cpt_and_audio_datasets():
# LoftQ alongside DoRA so a gate that refuses nothing cannot pass.
from utils.hardware import hardware
@@ -1847,6 +1876,8 @@ def test_mlx_start_accepts_dora():
with patch.object(hardware, "DEVICE", hardware.DeviceType.MLX):
route._validate_training_platform(_request(use_dora = True))
route._validate_training_platform(_request(is_decision = True))
+ route._validate_training_platform(_request(training_type = "Continued Pretraining"))
+ route._validate_training_platform(_request(is_dataset_audio = True))
with pytest.raises(HTTPException) as exc_info:
route._validate_training_platform(_request(use_loftq = True))
@@ -1857,7 +1888,7 @@ def test_mlx_start_detects_hardware_before_platform_validation():
from utils.hardware import hardware
route = _load_route_module("training_route_mlx_warm_platform_reject")
- request = _request(training_type = "Continued Pretraining")
+ request = _request(use_loftq = True)
detection_calls = 0
def detect_mlx():
@@ -1876,7 +1907,7 @@ def test_mlx_start_detects_hardware_before_platform_validation():
assert detection_calls == 1
assert exc_info.value.status_code == 400
- assert "Continued Pretraining" in exc_info.value.detail
+ assert "LoftQ" in exc_info.value.detail
def test_route_forwards_cache_reference_fields():
@@ -2419,7 +2450,7 @@ def test_pending_hub_model_resume_before_attestation_is_allowed():
},
}
- actual_repo_id, requires_exact_model, requires_exact_dataset, _, _ = (
+ actual_repo_id, requires_exact_model, requires_exact_dataset, _, _, _ = (
route._prepare_resume_resource_provenance(request, resume_run)
)
@@ -2901,8 +2932,9 @@ def test_worker_rejects_inference_only_model_formats():
assert (
worker._untrainable_model_format_error({**lora_adapter, runner_flag: True}) is not None
)
- with patch.object(worker, "_uses_mlx_training_backend", return_value = True):
- assert worker._untrainable_model_format_error(lora_adapter) is not None
+ # The MLX runner continues a saved adapter too.
+ with patch("core.training.training.should_use_mlx_training_backend", return_value = True):
+ assert worker._untrainable_model_format_error(lora_adapter) is None
assert worker._untrainable_model_format_error({"model_format": "safetensors"}) is None
assert worker._untrainable_model_format_error({}) is None
@@ -3541,6 +3573,78 @@ def test_worker_cached_auto_eval_without_split_metadata_stays_offline(monkeypatc
assert warnings == []
[email protected]("eval_steps", [0, 0.1])
+def test_mlx_streaming_loads_lazily_and_caps_a_separate_eval_split(eval_steps):
+ from core.training import worker
+ from core.training.eval_dataset import STREAMING_EVAL_MAX_SAMPLES
+
+ class _Stream:
+ def __init__(self, split):
+ self.split, self.taken = split, None
+
+ def take(self, count):
+ self.taken = count
+ return self
+
+ loads: list[dict] = []
+
+ def load_dataset(path, **kwargs):
+ loads.append({"path": path, **kwargs})
+ return _Stream(kwargs["split"])
+
+ config = {
+ "hf_dataset": "org/dataset",
+ "subset": "sub",
+ "train_split": "train",
+ "eval_split": "validation",
+ "eval_steps": eval_steps,
+ "dataset_streaming": True,
+ }
+
+ dataset, eval_dataset = worker._load_hf_train_and_eval_datasets(
+ config, None, load_dataset, lambda _message: None
+ )
+
+ assert dataset.split == "train" and dataset.taken is None
+ assert all(load["streaming"] is True and load["name"] == "sub" for load in loads)
+ if eval_steps:
+ assert eval_dataset.split == "validation"
+ assert eval_dataset.taken == STREAMING_EVAL_MAX_SAMPLES
+ else:
+ assert eval_dataset is None and len(loads) == 1
+
+
+class _SliceStream:
+ def __init__(self):
+ self.calls = []
+
+ def skip(self, count):
+ self.calls.append(("skip", count))
+ return self
+
+ def take(self, count):
+ self.calls.append(("take", count))
+ return self
+
+
+ ("start", "end", "calls"),
+ [
+ (None, None, []),
+ (3, None, [("skip", 3)]),
+ (None, 9, [("take", 10)]),
+ (5, 9, [("skip", 5), ("take", 5)]),
+ (5, 3, [("skip", 5), ("take", 0)]),
+ ],
+)
+def test_mlx_streaming_slice_is_lazy(start, end, calls):
+ from core.training import worker
+
+ stream = _SliceStream()
+ assert worker._slice_streaming_dataset(stream, start, end) is stream
+ assert stream.calls == calls
+
+
def test_worker_model_retry_refreshes_tokenizer_before_dataset():
from core.training import worker
diff --git a/studio/backend/tests/test_training_finetune_targets_matrix.py b/studio/backend/tests/test_training_finetune_targets_matrix.py
index ba6476a2ffe..368ffd9d00d 100644
--- a/studio/backend/tests/test_training_finetune_targets_matrix.py
+++ b/studio/backend/tests/test_training_finetune_targets_matrix.py
@@ -365,8 +365,8 @@ def test_pre_detect_training_model_leaves_a_valid_run_alone():
def test_mlx_worker_calls_the_guard_in_its_lora_branch():
- """_run_mlx_training imports mlx, so it cannot be invoked off Apple Silicon. Pin the call
- site structurally instead: inside `if use_lora:` and above the from_pretrained below it."""
+ """_run_mlx_training imports mlx, so it cannot be invoked off Apple Silicon. Run its guard
+ statement alone instead, and pin it above the from_pretrained below it."""
source = textwrap.dedent(inspect.getsource(_run_mlx_training))
tree = ast.parse(source)
calls = [
@@ -379,16 +379,36 @@ def test_mlx_worker_calls_the_guard_in_its_lora_branch():
assert len(calls) == 1
- guarded = [
+ guard = next(
node
- for node in ast.walk(tree)
- if isinstance(node, ast.If)
- and isinstance(node.test, ast.Name)
- and node.test.id == "use_lora"
- and any(call in ast.walk(node) for call in calls)
- ]
+ for node in tree.body[0].body
+ if isinstance(node, ast.If) and any(call in ast.walk(node) for call in calls)
+ )
+ code = compile(ast.Module(body = [guard], type_ignores = []), "<guard>", "exec")
+
+ def guarded_call(
+ use_lora,
+ model_format = None,
+ is_cpt = False,
+ ):
+ checked = []
+ exec(
+ code,
+ {
+ "use_lora": use_lora,
+ "is_cpt": is_cpt,
+ "config": {"model_format": model_format},
+ "_check_mlx_finetune_targets": checked.append,
+ },
+ )
+ return bool(checked)
- assert guarded, "_check_mlx_finetune_targets must sit under `if use_lora:`"
+ assert guarded_call(True)
+ assert not guarded_call(False)
+ # A continued adapter keeps its own targets, so the requested ones are not checked.
+ assert not guarded_call(True, model_format = "adapter")
+ # CPT builds its adapter from target_modules alone, as on CUDA.
+ assert not guarded_call(True, is_cpt = True)
from_pretrained_lines = [
node.lineno
@@ -397,7 +417,6 @@ def test_mlx_worker_calls_the_guard_in_its_lora_branch():
and isinstance(node.func, ast.Attribute)
and node.func.attr == "from_pretrained"
]
-
assert from_pretrained_lines
assert calls[0].lineno < min(from_pretrained_lines)
@@ -558,8 +577,12 @@ def test_the_effective_check_runs_after_the_back_fill_at_the_mlx_call_site():
source = inspect.getsource(worker)
call = source.index("_check_mlx_effective_targets(\n config,")
backfill = source.index(" finetune_language = True")
- forced = source.index('config.get("finetune_vision_layers", False) if is_vlm else False')
+ forced = source.index(
+ 'config.get("finetune_vision_layers", False) if is_vlm and is_dataset_image else False'
+ )
assert (
forced < backfill < call
), "the effective check must follow both the is_vlm narrowing and the back-fill"
- assert call < source.index("FastMLXModel.get_peft_model("), "and precede the loader"
+ assert call < source.index(
+ "FastMLXModel.get_peft_model(model, **peft_kwargs)"
+ ), "and precede the loader"
diff --git a/studio/backend/tests/test_training_provenance.py b/studio/backend/tests/test_training_provenance.py
index ab57f9b319a..69ad830e671 100644
--- a/studio/backend/tests/test_training_provenance.py
+++ b/studio/backend/tests/test_training_provenance.py
@@ -23,6 +23,7 @@ from core.training.provenance import (
exact_resume_resource_requirements,
exact_dataset_snapshot_path,
exact_model_snapshot_path,
+ local_model_resume_expectation,
normalize_worker_provenance_event,
resource_provenance_allows_resume,
validate_exact_model_pin,
@@ -327,7 +328,7 @@ def test_resume_route_restores_attested_actual_model_repo(tmp_path):
hf_dataset = "org/dataset",
)
- actual_repo_id, _, _, _, _ = route._prepare_resume_resource_provenance(
+ actual_repo_id, _, _, _, _, _ = route._prepare_resume_resource_provenance(
request,
{
"model_name": "org/selected",
@@ -1647,3 +1648,361 @@ def test_provenance_continues_after_failed_finalization(tmp_path):
assert backend._run_finalized is False
assert backend._db_config[RESOURCE_PROVENANCE_KEY]["status"] == "complete"
update.assert_called_once()
+
+
+def _continued_adapter_run(tmp_path):
+ adapter = tmp_path / "outputs" / "adapter"
+ adapter.mkdir(parents = True)
+ base = _model_snapshot(tmp_path, "org/base", "base-commit")
+ dataset = _dataset_snapshot(tmp_path, "org/dataset", "dataset-commit")
+ source_config = {
+ "model_name": str(adapter),
+ "actual_model_repo_id": "org/base",
+ "model_snapshot_path": str(base),
+ "hf_dataset": "org/dataset",
+ "dataset_snapshot_path": str(dataset),
+ "training_type": "LoRA/QLoRA",
+ "format_type": "alpaca",
+ "load_in_4bit": False,
+ RESOURCE_PROVENANCE_KEY: {
+ "version": 1,
+ "status": "complete",
+ "model_status": "attested",
+ "model_load_mode": "unquantized",
+ "dataset_status": "attested",
+ "reasons": [],
+ },
+ }
+ return adapter, base, dataset, source_config
+
+
+def test_a_continued_adapter_run_resumes_with_its_base_checked_after_load(tmp_path):
+ from core.training import worker
+ from core.training.training import _apply_cache_pins, _build_training_worker_config
+ from models.training import TrainingStartRequest
+
+ route = _load_training_route()
+ adapter, base, dataset, source_config = _continued_adapter_run(tmp_path)
+ request = TrainingStartRequest(
+ model_name = str(adapter),
+ training_type = "LoRA/QLoRA",
+ format_type = "alpaca",
+ hf_dataset = "org/dataset",
+ load_in_4bit = False,
+ )
+
+ actual, exact_model, exact_dataset, exact_all, load_mode, overrides = (
+ route._prepare_resume_resource_provenance(
+ request, {"model_name": str(adapter), "config_json": source_config}
+ )
+ )
+
+ # The base is not pinned as the load target, or the bare base would load without the adapter.
+ assert (actual, exact_model, exact_dataset, exact_all, load_mode) == (
+ None,
+ False,
+ True,
+ False,
+ None,
+ )
+ expected = {"repo_id": "org/base", "snapshot_path": str(base), "load_mode": "unquantized"}
+ assert overrides == {"resume_expected_model": expected}
+ # Kwargs shaped like the route's, through the real builder and pin steps.
+ config = _build_training_worker_config(
+ {
+ **request.model_dump(),
+ "actual_model_repo_id": actual,
+ "resume_from_checkpoint": str(tmp_path / "outputs" / "run" / "checkpoint-5"),
+ "require_exact_resume_resources": exact_all,
+ "require_exact_model_resource": exact_model,
+ "require_exact_dataset_resource": exact_dataset,
+ **overrides,
+ }
+ )
+ _apply_cache_pins(config)
+ events: list[dict] = []
+ assert worker._verify_config_pins(config, SimpleNamespace(put = events.append)) is True, events
+ assert config["model_snapshot_path"] is None
+ assert config["dataset_snapshot_path"] == str(dataset.resolve())
+ assert config["resume_expected_model"] == expected
+
+ # Refused for a moved base, the resume records its dataset alone, so a retry checks the same base.
+ from core.training.provenance import initialize_resource_provenance
+ from core.training.training import _sanitize_db_config
+
+ _model_snapshot(tmp_path, "org/base", "newer-commit")
+ moved = SimpleNamespace(
+ config = SimpleNamespace(_name_or_path = "org/base", _commit_hash = "newer-commit")
+ )
+ initialize_resource_provenance(config)
+ events = []
+ assert not worker._emit_resource_provenance(
+ SimpleNamespace(put = events.append),
+ config,
+ moved,
+ model_load_target = str(adapter),
+ model_load_in_4bit = False,
+ dataset_loaded_from_exact_snapshot = True,
+ )
+ row = {**_sanitize_db_config(config), **normalize_worker_provenance_event(events[0], config)}
+ *_, retry = route._prepare_resume_resource_provenance(
+ request, {"model_name": str(adapter), "config_json": row}
+ )
+ assert retry == overrides
+
+
+ "loaded_commit, quantized_base, expected_error",
+ [
+ ("base-commit", False, None),
+ ("newer-commit", False, "no longer resolves"),
+ ("base-commit", True, "trained on a 'unquantized' load"),
+ ],
+)
+def test_a_resumed_adapter_run_stops_unless_it_loaded_the_base_it_trained_on(
+ tmp_path, loaded_commit, quantized_base, expected_error
+):
+ from core.training import worker
+
+ adapter, base, _dataset, source_config = _continued_adapter_run(tmp_path)
+ if loaded_commit != "base-commit":
+ _model_snapshot(tmp_path, "org/base", loaded_commit)
+ if quantized_base:
+ (base / "config.json").write_text(
+ json.dumps({"quantization_config": {"load_in_4bit": True}}), encoding = "utf-8"
+ )
+ model = SimpleNamespace(
+ config = SimpleNamespace(_name_or_path = "org/base", _commit_hash = loaded_commit)
+ )
+ config = {
+ **source_config,
+ "resume_expected_model": local_model_resume_expectation(source_config),
+ }
+ events: list[dict] = []
+
+ ok = worker._emit_resource_provenance(
+ SimpleNamespace(put = events.append),
+ config,
+ model,
+ model_load_target = str(adapter),
+ model_load_in_4bit = quantized_base,
+ dataset_loaded_from_exact_snapshot = False,
+ )
+
+ assert ok is (expected_error is None)
+ assert [event["type"] for event in events] == (
+ ["resource_provenance"] if expected_error is None else ["resource_provenance", "error"]
+ )
+ # A refused load's base is not recorded, or it would become the next resume's reference.
+ assert (events[0]["model"]["status"] == "attested") is (expected_error is None)
+ assert expected_error is None or expected_error in events[1]["error"]
+
+
+def test_a_continued_adapter_resume_under_the_latest_runtime_is_refused_before_loading(tmp_path):
+ expected = {"repo_id": "org/base", "snapshot_path": "/base", "load_mode": "runtime_4bit"}
+ # A failed resume's row, whose only check is the expectation it carries.
+ stored = {
+ "model_name": str(tmp_path),
+ "load_in_4bit": True,
+ "resume_expected_model": expected,
+ RESOURCE_PROVENANCE_KEY: {"version": 1, "status": "pending"},
+ }
+ assert exact_resume_requires_current_4bit(stored) is True
+
+ with patch("utils.transformers_version.latest_tier_active_for", return_value = True):
+ with pytest.raises(ExactResumeResourcesUnavailable, match = "original 4-bit"):
+ effective_training_load_in_4bit(
+ {"load_in_4bit": True, "resume_expected_model": expected}, "/adapter", None
+ )
+
+
+def test_only_a_local_model_run_gets_a_loaded_base_expectation(tmp_path):
+ _adapter, _base, _dataset, source_config = _continued_adapter_run(tmp_path)
+
+ assert local_model_resume_expectation({**source_config, "model_name": "org/base"}) is None
+ assert local_model_resume_expectation({**source_config, "actual_model_repo_id": None}) is None
+
+
+_STREAM_COMMIT = "0123456789abcdef0123456789abcdef01234567"
+
+
+def _streaming_config(revision = _STREAM_COMMIT):
+ return {
+ "model_name": "org/model",
+ "hf_dataset": "org/dataset",
+ "dataset_streaming": True,
+ "dataset_revision": revision,
+ "load_in_4bit": False,
+ }
+
+
+ "revision, attested",
+ [(_STREAM_COMMIT, True), (None, False), ("main", False)],
+)
+def test_a_stream_is_attested_by_the_commit_it_was_pinned_to(tmp_path, revision, attested):
+ model = _model_snapshot(tmp_path, "org/model", "model-commit")
+ config = {**_streaming_config(revision), "model_snapshot_path": str(model)}
+
+ event = build_worker_provenance_event(
+ config,
+ object(),
+ model_load_target = str(model),
+ model_load_in_4bit = False,
+ dataset_loaded_from_exact_snapshot = False,
+ )
+ updates = normalize_worker_provenance_event(event, config)
+
+ marker = updates[RESOURCE_PROVENANCE_KEY]
+ assert marker["dataset_status"] == ("attested" if attested else "incomplete")
+ assert marker["status"] == ("complete" if attested else "incomplete")
+ assert ("dataset_streaming_unattested" in marker["reasons"]) is not attested
+
+
+def test_a_stream_attests_only_the_commit_this_process_pinned(tmp_path):
+ model = _model_snapshot(tmp_path, "org/model", "model-commit")
+ config = {**_streaming_config(), "model_snapshot_path": str(model)}
+ event = build_worker_provenance_event(
+ config,
+ object(),
+ model_load_target = str(model),
+ model_load_in_4bit = False,
+ dataset_loaded_from_exact_snapshot = False,
+ )
+ event["dataset"]["revision"] = "f" * 40
+
+ marker = normalize_worker_provenance_event(event, config)[RESOURCE_PROVENANCE_KEY]
+
+ assert marker["dataset_status"] == "incomplete"
+
+
[email protected]("revision", [_STREAM_COMMIT, None])
+def test_a_pinned_stream_resumes_at_its_commit(tmp_path, revision):
+ from fastapi import HTTPException
+
+ from core.training import worker
+ from core.training.training import _apply_cache_pins, _build_training_worker_config
+ from models.training import TrainingStartRequest
+
+ route = _load_training_route()
+ model = _model_snapshot(tmp_path, "org/model", "model-commit")
+ source_config = {
+ **_streaming_config(revision),
+ "actual_model_repo_id": "org/model",
+ "model_snapshot_path": str(model),
+ "training_type": "LoRA/QLoRA",
+ "format_type": "alpaca",
+ "max_steps": 10,
+ RESOURCE_PROVENANCE_KEY: {
+ "version": 1,
+ "status": "complete",
+ "model_status": "attested",
+ "model_load_mode": "unquantized",
+ "dataset_status": "attested",
+ "reasons": [],
+ },
+ }
+ request = TrainingStartRequest(
+ model_name = "org/model",
+ training_type = "LoRA/QLoRA",
+ format_type = "alpaca",
+ hf_dataset = "org/dataset",
+ dataset_streaming = True,
+ max_steps = 10,
+ )
+ if revision is None:
+ with pytest.raises(HTTPException, match = "exact snapshots are no longer available"):
+ route._prepare_resume_resource_provenance(
+ request, {"model_name": "org/model", "config_json": source_config}
+ )
+ return
+
+ repo, exact_model, exact_dataset, exact_all, mode, overrides = (
+ route._prepare_resume_resource_provenance(
+ request, {"model_name": "org/model", "config_json": source_config}
+ )
+ )
+
+ assert overrides == {"dataset_revision": _STREAM_COMMIT}
+ config = _build_training_worker_config(
+ {
+ "model_name": "org/model",
+ "hf_dataset": "org/dataset",
+ "dataset_streaming": True,
+ "model_snapshot_path": request.model_snapshot_path,
+ "actual_model_repo_id": repo,
+ "resume_model_load_mode": mode,
+ "resume_from_checkpoint": "/outputs/run/checkpoint-5",
+ "require_exact_resume_resources": exact_all,
+ "require_exact_model_resource": exact_model,
+ "require_exact_dataset_resource": exact_dataset,
+ "load_in_4bit": False,
+ **overrides,
+ }
+ )
+ _apply_cache_pins(config)
+ events: list[dict] = []
+ assert worker._verify_config_pins(config, SimpleNamespace(put = events.append)) is True, events
+ assert config["dataset_revision"] == _STREAM_COMMIT
+
+
[email protected]("hub_commit", [_STREAM_COMMIT, None])
+def test_a_new_stream_is_pinned_to_the_current_hub_commit(monkeypatch, hub_commit):
+ import huggingface_hub
+ from core.training.training import _apply_cache_pins
+
+ def _info(self, repo_id, **kw):
+ if hub_commit is None:
+ raise ConnectionError("hub unreachable")
+ return SimpleNamespace(sha = hub_commit)
+
+ monkeypatch.setattr(huggingface_hub.HfApi, "dataset_info", _info)
+ config = {"model_name": "org/model", "hf_dataset": "org/dataset", "dataset_streaming": True}
+
+ _apply_cache_pins(config)
+
+ assert config["dataset_revision"] == hub_commit
+ assert config["dataset_snapshot_path"] is None
+ assert any("cannot be resumed" in m for m in config["cache_pin_warnings"]) is (
+ hub_commit is None
+ )
+
+
+def test_a_resume_that_dies_before_attesting_hands_on_its_checks(tmp_path):
+ from core.training.provenance import initialize_resource_provenance
+ from core.training.training import (
+ _apply_cache_pins,
+ _build_training_worker_config,
+ _sanitize_db_config,
+ )
+ from models.training import TrainingStartRequest
+
+ route = _load_training_route()
+ adapter, _base, _dataset, source_config = _continued_adapter_run(tmp_path)
+ source_config = {**source_config, **_streaming_config(), "model_name": str(adapter)}
+ source_config.pop("dataset_snapshot_path")
+ request = TrainingStartRequest(
+ model_name = str(adapter),
+ training_type = "LoRA/QLoRA",
+ format_type = "alpaca",
+ hf_dataset = "org/dataset",
+ dataset_streaming = True,
+ load_in_4bit = False,
+ )
+
+ def resume_from(stored):
+ *_, overrides = route._prepare_resume_resource_provenance(
+ request, {"model_name": str(adapter), "config_json": stored}
+ )
+ return overrides
+
+ first = resume_from(source_config)
+ assert set(first) == {"dataset_revision", "resume_expected_model"}
+ config = _build_training_worker_config(
+ {**request.model_dump(), "resume_from_checkpoint": "/outputs/run/checkpoint-5", **first}
+ )
+ _apply_cache_pins(config)
+ initialize_resource_provenance(config)
+
+ # The row a resume leaves when it is refused or dies before its worker attests.
+ assert resume_from(_sanitize_db_config(config)) == first
diff --git a/studio/backend/tests/test_training_streaming_mlx_warm.py b/studio/backend/tests/test_training_streaming_mlx_warm.py
index 4060dc5edd7..f9a3bc1f732 100644
--- a/studio/backend/tests/test_training_streaming_mlx_warm.py
+++ b/studio/backend/tests/test_training_streaming_mlx_warm.py
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-"""The MLX streaming guard must force detection first, since DEVICE is None during warm-up."""
+"""A streaming start must force detection first, since DEVICE is None during warm-up."""
from __future__ import annotations
@@ -16,8 +16,7 @@ import routes.training as training_routes
from auth.authentication import authenticated_via_api_key, get_current_subject
from utils.hardware import hardware as hw
-# Clears every streaming precondition but the MLX guard; load_in_4bit off keeps the
-# sidecar probe offline.
+# Clears every streaming precondition; load_in_4bit off keeps the sidecar probe offline.
_STREAMING_START = {
"model_name": "unsloth/Llama-3.2-1B-Instruct",
"training_type": "LoRA/QLoRA",
@@ -35,8 +34,6 @@ _NON_STREAMING_START = {
"max_steps": None,
}
-_MLX_REJECTION = "dataset_streaming is not yet supported on Apple Silicon (MLX)"
-
@pytest.fixture
def hardware_globals():
@@ -99,36 +96,18 @@ def _pretend_cpu_linux(monkeypatch):
monkeypatch.setattr(hw, "_has_usable_mlx_stack", lambda: False)
-def test_streaming_is_rejected_on_mlx_before_detection_has_run(
- monkeypatch, hardware_globals, spawn_calls, client
-):
- """The regression: DEVICE is still None when the request lands."""
[email protected]("detected", [False, True], ids = ["warm-window", "detected"])
+def test_streaming_starts_on_mlx(monkeypatch, hardware_globals, spawn_calls, client, detected):
+ """The MLX worker streams too, whether or not the warm thread has detected the device yet."""
_pretend_apple_silicon(monkeypatch)
- hardware_globals.DEVICE = None # warm thread has not finished
+ hardware_globals.DEVICE = hw.DeviceType.MLX if detected else None
response = client.post("/training/start", json = _STREAMING_START)
- assert response.status_code == 400, (
- "streaming start on an Apple Silicon host was not rejected while DEVICE "
- f"was still unset (got {response.status_code}: {response.text}); the guard "
- "read the pre-detection default instead of detecting"
- )
- assert _MLX_REJECTION in response.json()["detail"]
- spawn_calls.start_training.assert_not_called()
-
-
-def test_the_guard_detects_rather_than_reading_the_default(
- monkeypatch, hardware_globals, spawn_calls, client
-):
- """The guard forces detection, not some later step: DEVICE goes None -> MLX across
- a request that never reaches the worker config builder."""
- _pretend_apple_silicon(monkeypatch)
- hardware_globals.DEVICE = None
-
- client.post("/training/start", json = _STREAMING_START)
-
+ assert response.status_code == 200, response.text
+ assert response.json()["status"] == "queued"
assert hardware_globals.DEVICE == hw.DeviceType.MLX
- spawn_calls.start_training.assert_not_called()
+ spawn_calls.start_training.assert_called_once()
def test_streaming_still_starts_on_a_non_mlx_host_during_the_warm_window(
@@ -146,20 +125,6 @@ def test_streaming_still_starts_on_a_non_mlx_host_during_the_warm_window(
spawn_calls.start_training.assert_called_once()
-def test_rejection_still_fires_once_detection_has_already_run(
- monkeypatch, hardware_globals, spawn_calls, client
-):
- """The pre-existing behaviour, unchanged: DEVICE already MLX."""
- _pretend_apple_silicon(monkeypatch)
- hardware_globals.DEVICE = hw.DeviceType.MLX
-
- response = client.post("/training/start", json = _STREAMING_START)
-
- assert response.status_code == 400
- assert _MLX_REJECTION in response.json()["detail"]
- spawn_calls.start_training.assert_not_called()
-
-
def test_non_streaming_start_detects_before_entering_the_sync_backend(
monkeypatch, hardware_globals, spawn_calls, client
):
diff --git a/studio/backend/utils/audio_tokens.py b/studio/backend/utils/audio_tokens.py
index 79eb6327e8a..54681067da7 100644
--- a/studio/backend/utils/audio_tokens.py
+++ b/studio/backend/utils/audio_tokens.py
@@ -92,7 +92,15 @@ _MAX_TOKENIZER_CONFIG_BYTES = 32 * 1024 * 1024
def may_hold_audio_tokens(raw: str) -> bool:
"""Whether a tokenizer_config's raw text is worth parsing. A false True costs only
the parse that would have happened anyway; a false False misclassifies."""
- return any(marker in raw for marker in AUDIO_TOKEN_MARKERS)
+ # '"audio_token"' too: a JSON-escaped placeholder hides the markers from this raw scan.
+ return '"audio_token"' in raw or any(marker in raw for marker in AUDIO_TOKEN_MARKERS)
+
+
+def declares_audio_token(tok_config: dict) -> bool:
+ """Whether a transformers 5 tokenizer_config names an audio placeholder in its top-level
+ audio_token (Gemma 4) instead of added_tokens_decoder."""
+ token = tok_config.get("audio_token")
+ return isinstance(token, str) and bool(token)
def classify_audio_tokens(tok_config: dict) -> Optional[str]:
diff --git a/studio/backend/utils/datasets/audio_rows.py b/studio/backend/utils/datasets/audio_rows.py
new file mode 100644
index 00000000000..a291187747f
--- /dev/null
+++ b/studio/backend/utils/datasets/audio_rows.py
@@ -0,0 +1,230 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Audio dataset helpers shared by the CUDA and MLX trainers; torch-free."""
+
+from typing import Any
+
+
+def resolve_audio_columns(dataset, custom_format_mapping: dict = None) -> dict:
+ """Resolve audio/text/speaker columns from user mapping or fallback. Returns dict with keys
+ audio_col, text_col, speaker_col (may be None)."""
+ cols = dataset.column_names
+
+ if custom_format_mapping:
+ audio_col = None
+ text_col = None
+ speaker_col = None
+ for col, role in custom_format_mapping.items():
+ if role == "audio":
+ audio_col = col
+ elif role == "text":
+ text_col = col
+ elif role == "speaker_id":
+ speaker_col = col
+ if audio_col and audio_col in cols and text_col and text_col in cols:
+ return {
+ "audio_col": audio_col,
+ "text_col": text_col,
+ "speaker_col": speaker_col,
+ }
+
+ audio_col = next((c for c in cols if c.lower() in ("audio", "speech")), None)
+ text_col = next(
+ (c for c in cols if c.lower() in ("text", "sentence", "transcript", "transcription")),
+ None,
+ )
+
+ speaker_col = None
+ if "source" in cols:
+ speaker_col = "source"
+ elif "speaker_id" in cols:
+ speaker_col = "speaker_id"
+
+ if audio_col is None or text_col is None or speaker_col is None:
+ from hub.utils.dataset_format import detect_multimodal_dataset
+
+ detected = detect_multimodal_dataset(dataset)
+ audio_col = audio_col or detected["detected_audio_column"]
+ text_col = text_col or detected["detected_text_column"]
+ speaker_col = speaker_col or detected["detected_speaker_column"]
+
+ return {
+ "audio_col": audio_col,
+ "text_col": text_col,
+ "speaker_col": speaker_col,
+ }
+
+
+def image_slot_count(row_images: Any) -> int:
+ if isinstance(row_images, list):
+ return sum(1 for image in row_images if image is not None)
+ return 0 if row_images is None else 1
+
+
+def audio_transcription_messages(
+ audio: Any,
+ label: str,
+ n_images: int = 0,
+) -> list:
+ """System/user/assistant chat turns that train an audio-input model to transcribe ``audio``,
+ with one image slot per image of the row ahead of the audio."""
+ return [
+ {
+ "role": "system",
+ "content": [
+ {
+ "type": "text",
+ "text": "You are an assistant that transcribes speech accurately.",
+ }
+ ],
+ },
+ {
+ "role": "user",
+ "content": [
+ *({"type": "image"} for _ in range(n_images)),
+ {"type": "audio", "audio": audio},
+ {"type": "text", "text": "Please transcribe this audio."},
+ ],
+ },
+ {"role": "assistant", "content": [{"type": "text", "text": label}]},
+ ]
+
+
+def resolve_audio_vlm_image_column(
+ dataset,
+ custom_format_mapping,
+ audio_col,
+ text_col = None,
+ speaker_col = None,
+):
+ cols = dataset.column_names
+ reserved = {audio_col, text_col}
+ reserved.update(
+ col for col, role in (custom_format_mapping or {}).items() if role and role != "image"
+ )
+ for col, role in (custom_format_mapping or {}).items():
+ if role == "image" and col in cols and col not in reserved:
+ return col
+ from hub.utils.dataset_format import (
+ _first_row,
+ _is_image_list_item,
+ _is_image_value,
+ detect_multimodal_dataset,
+ )
+
+ from datasets import Image
+
+ def image_typed(col):
+ feature = (getattr(dataset, "features", None) or {}).get(col)
+ inner = feature[0] if isinstance(feature, list) and feature else feature
+ return isinstance(getattr(inner, "feature", inner), Image)
+
+ if speaker_col and not image_typed(speaker_col):
+ reserved.add(speaker_col)
+
+ def image_list(value):
+ if not isinstance(value, list):
+ return False
+ items = [
+ item for item in value if item is not None and not (isinstance(item, str) and not item)
+ ]
+ return bool(items) and all(map(_is_image_list_item, items))
+
+ sample = _first_row(dataset) or {}
+
+ def empty(value):
+ # Not `in (None, "", [])`: that compares a pixel ndarray elementwise and raises.
+ return (
+ value is None
+ or (isinstance(value, str) and not value)
+ or (
+ isinstance(value, list)
+ and all(item is None or (isinstance(item, str) and not item) for item in value)
+ )
+ )
+
+ ordered = list(dict.fromkeys(detect_multimodal_dataset(dataset)["multimodal_columns"] + cols))
+ ordered = [col for col in ordered if col not in reserved]
+ typed = {col: image_typed(col) for col in ordered}
+ values = {col: sample.get(col) for col in ordered}
+ first_candidate = next(
+ (
+ i
+ for i, col in enumerate(ordered)
+ if typed[col] or _is_image_value(values[col]) or image_list(values[col])
+ ),
+ len(ordered),
+ )
+ unresolved = {col for col in ordered[:first_candidate] if not typed[col] and empty(values[col])}
+ if unresolved:
+ for i, row in enumerate(dataset):
+ if i >= 100 or not unresolved:
+ break
+ for col in tuple(unresolved):
+ if not empty(row.get(col)):
+ values[col] = row[col]
+ unresolved.remove(col)
+ return next(
+ (
+ col
+ for col in ordered
+ if typed[col] or _is_image_value(values[col]) or image_list(values[col])
+ ),
+ None,
+ )
+
+
+def prepare_audio_vlm_image_column(dataset, image_col):
+ from datasets import Image, Sequence
+
+ feature = dataset.features.get(image_col)
+ inner = feature[0] if isinstance(feature, list) else getattr(feature, "feature", None)
+
+ pixel_feature, dimensions = feature, 0
+ while getattr(pixel_feature, "feature", None) is not None:
+ pixel_feature = pixel_feature.feature
+ dimensions += 1
+ if dimensions <= 1 and getattr(pixel_feature, "dtype", "") in (
+ "string",
+ "large_string",
+ ):
+
+ def normalize_paths(values):
+ return {
+ image_col: [
+ [None if item == "" else item for item in value]
+ if isinstance(value, list)
+ else None
+ if value == ""
+ else value
+ for value in values
+ ]
+ }
+
+ dataset = dataset.map(normalize_paths, batched = True, input_columns = image_col)
+ if dimensions in (2, 3) and getattr(pixel_feature, "dtype", "").startswith(
+ ("int", "uint", "float")
+ ):
+ from itertools import islice
+
+ import numpy as np
+
+ value = next(
+ (row[image_col] for row in islice(dataset, 100) if row[image_col] is not None),
+ None,
+ )
+ shape = np.asarray(value).shape if value is not None else ()
+ if value is None or len(shape) == 2 or (len(shape) == 3 and shape[-1] in (1, 3, 4)):
+ inner = None
+
+ # Image(decode = False) yields {"path", "bytes"} dicts the processor rejects.
+ def decodes(f):
+ return isinstance(f, Image) and f.decode
+
+ if inner is not None:
+ if not decodes(inner):
+ dataset = dataset.cast_column(image_col, Sequence(Image()))
+ elif not decodes(feature):
+ dataset = dataset.cast_column(image_col, Image())
+ return dataset
diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py
index 4802abef13c..4fc3452d260 100644
--- a/studio/backend/utils/models/model_config.py
+++ b/studio/backend/utils/models/model_config.py
@@ -45,6 +45,7 @@ from utils.training_runs import (
from utils.audio_tokens import (
AUDIO_TOKENIZER_CONFIG_PATHS as _AUDIO_TOKENIZER_CONFIG_PATHS,
classify_audio_tokens as _check_token_patterns,
+ declares_audio_token as _declares_audio_token,
is_audio_input_type,
may_hold_audio_tokens as _may_hold_audio_tokens,
)
@@ -1820,6 +1821,32 @@ def _detect_audio_from_config(
return None
+def _read_local_model_config(root: Path, missing_is_absent: bool) -> Optional[dict]:
+ """None when config.json cannot be read. A cache snapshot can hold the tokenizer without
+ it (a tokenizer-only download), so there a missing file is unknown, not absent, unless the
+ snapshot is an adapter repo, which never ships one."""
+ config_file = root / "config.json"
+ if not config_file.is_file():
+ return {} if missing_is_absent or (root / "adapter_config.json").is_file() else None
+ try:
+ config = json.loads(config_file.read_text(encoding = "utf-8-sig"))
+ except Exception:
+ return None
+ return config if isinstance(config, dict) else {}
+
+
+def _classify_audio_tokenizer(tok_config: dict, read_model_config) -> Tuple[Optional[str], bool]:
+ """(audio_type, known). A declared audio_token is audio input only with an audio encoder:
+ Gemma 4 26B/31B carry the token and none, so config.json decides, read only for that case."""
+ result = _check_token_patterns(tok_config)
+ if result or not _declares_audio_token(tok_config):
+ return result, True
+ model_config = read_model_config()
+ if model_config is None:
+ return None, False
+ return ("audio_vlm" if model_config.get("audio_config") else None), True
+
+
def _detect_audio_from_tokenizer(
model_name: str,
hf_token: Optional[str] = None,
@@ -1829,6 +1856,8 @@ def _detect_audio_from_tokenizer(
"""Returns definitive False only on a transient read failure, so callers retry instead of caching."""
read_any = False # parsed at least one tokenizer_config -> a None is definitive
+ # A declared audio_token whose encoder config could not be read keeps every negative unknown.
+ unresolved = False
# 1) Selected local directory or local HF cache first (works for gated/offline models)
try:
@@ -1864,7 +1893,8 @@ def _detect_audio_from_tokenizer(
current: list = [] # resolved lazily: only a negative answer needs it
# Only a read standing in for the Hub copy has to prove the file whole.
- may_answer_for_hub = not local_files_only and not is_local_path(model_name)
+ local_model = is_local_path(model_name)
+ may_answer_for_hub = not local_files_only and not local_model
for root in roots:
root_read: set = set()
for tok_path in _AUDIO_TOKENIZER_CONFIG_PATHS:
@@ -1881,20 +1911,29 @@ def _detect_audio_from_tokenizer(
except Exception:
continue
# Raw scan misses escaped markers (Go's encoding/json writes them), so decode first.
- result = _check_token_patterns(decoded)
+ result, known = _classify_audio_tokenizer(
+ decoded, lambda: _read_local_model_config(root, local_model)
+ )
if result:
return result, True
+ if not known:
+ unresolved = True
+ continue
elif not raw.rstrip().endswith("}"):
continue
read_any = True
root_read.add(tok_path)
continue
- tok_config = json.loads(raw)
- read_any = True
- root_read.add(tok_path)
- result = _check_token_patterns(tok_config)
+ result, known = _classify_audio_tokenizer(
+ json.loads(raw), lambda: _read_local_model_config(root, local_model)
+ )
if result:
return result, True
+ if not known:
+ unresolved = True
+ continue
+ read_any = True
+ root_read.add(tok_path)
except Exception as e:
logger.debug(f"Could not read {tok_file} for {model_name}: {e}")
# Negatives are cached for the process, so only answer negatively after reading every path.
@@ -1917,17 +1956,17 @@ def _detect_audio_from_tokenizer(
# This raw requests.get ignores HF offline flags, so gate it explicitly.
if local_files_only or _env_offline():
- return None, read_any
+ return None, read_any and not unresolved
# A filesystem path is not a repo id, so no Hub URL; the local read is the whole answer, or /loras blocks the event loop on timeouts.
if is_local_path(model_name):
- return None, read_any
+ return None, read_any and not unresolved
try:
import requests
import os
except Exception:
- return None, read_any
+ return None, read_any and not unresolved
# `False` is falsy, so `or` would reach past it to the ambient credential.
token = None if is_anonymous(hf_token) else (hf_token or os.environ.get("HF_TOKEN"))
@@ -1940,6 +1979,21 @@ def _detect_audio_from_tokenizer(
from utils.hf_endpoint import get_hf_endpoint
hf_endpoint = get_hf_endpoint()
+
+ def _fetch_model_config() -> Optional[dict]:
+ url = f"{hf_endpoint}/{model_name}/resolve/{revision_path}/config.json"
+ try:
+ resp = requests.get(url, headers = headers, timeout = 15)
+ if resp.status_code == 404:
+ return {}
+ if not resp.ok:
+ return None
+ config = resp.json()
+ except Exception as e:
+ logger.debug(f"Could not fetch config.json for {model_name}: {e}")
+ return None
+ return config if isinstance(config, dict) else {}
+
for tok_path in _AUDIO_TOKENIZER_CONFIG_PATHS:
url = f"{hf_endpoint}/{model_name}/resolve/{revision_path}/{tok_path}"
try:
@@ -1959,13 +2013,16 @@ def _detect_audio_from_tokenizer(
logger.debug(f"Bad tokenizer_config for {model_name}/{tok_path}: {e}")
transient = True
continue
- read_any = True
- result = _check_token_patterns(tok_config)
+ result, known = _classify_audio_tokenizer(tok_config, _fetch_model_config)
if result:
return result, True
+ if not known:
+ unresolved = True
+ continue
+ read_any = True
# No audio tokens: definitive unless every attempt failed transiently.
- return None, (read_any or not transient)
+ return None, (read_any or not transient) and not unresolved
def _is_mmproj(filename: str) -> bool:
diff --git a/studio/frontend/src/features/studio/sections/dataset-advanced-settings-section.tsx b/studio/frontend/src/features/studio/sections/dataset-advanced-settings-section.tsx
index 56e39fd4361..cf2a382685a 100644
--- a/studio/frontend/src/features/studio/sections/dataset-advanced-settings-section.tsx
+++ b/studio/frontend/src/features/studio/sections/dataset-advanced-settings-section.tsx
@@ -1,7 +1,6 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-import { usePlatformStore } from "@/config/env";
import { useTrainingConfigStore } from "@/features/training";
import { useT } from "@/i18n";
import { toast } from "@/lib/toast";
@@ -12,7 +11,6 @@ import { getDatasetStreamingBlockers } from "./dataset-panel-helpers";
export function DatasetAdvancedSettingsSection() {
const t = useT();
- const platformDeviceType = usePlatformStore((state) => state.deviceType);
const {
datasetEvalSplit,
datasetFormat,
@@ -63,7 +61,6 @@ export function DatasetAdvancedSettingsSection() {
datasetSource,
datasetSplit,
evalSteps,
- isAppleSilicon: platformDeviceType === "mac",
isAudioModel,
isDatasetAudio,
isDatasetImage,
diff --git a/studio/frontend/src/features/studio/sections/dataset-advanced-settings.tsx b/studio/frontend/src/features/studio/sections/dataset-advanced-settings.tsx
index 55692242fc6..6fabc589fb8 100644
--- a/studio/frontend/src/features/studio/sections/dataset-advanced-settings.tsx
+++ b/studio/frontend/src/features/studio/sections/dataset-advanced-settings.tsx
@@ -46,7 +46,6 @@ const DATASET_STREAMING_BLOCKER_KEYS: Record<
embeddingModel: "studio.dataset.streaming.blockers.embeddingModel",
imageDataset: "studio.dataset.streaming.blockers.imageDataset",
audioDataset: "studio.dataset.streaming.blockers.audioDataset",
- appleSilicon: "studio.dataset.streaming.blockers.appleSilicon",
};
export function DatasetAdvancedSettings({
diff --git a/studio/frontend/src/features/studio/sections/dataset-panel-helpers.ts b/studio/frontend/src/features/studio/sections/dataset-panel-helpers.ts
index 2b16741b131..ce30e69afaf 100644
--- a/studio/frontend/src/features/studio/sections/dataset-panel-helpers.ts
+++ b/studio/frontend/src/features/studio/sections/dataset-panel-helpers.ts
@@ -15,8 +15,7 @@ export type DatasetStreamingBlocker =
| "audioModel"
| "embeddingModel"
| "imageDataset"
- | "audioDataset"
- | "appleSilicon";
+ | "audioDataset";
export function getFileExtension(fileName: string): string {
const extensionStart = fileName.lastIndexOf(".");
@@ -42,7 +41,6 @@ export function getDatasetStreamingBlockers({
datasetSource,
datasetSplit,
evalSteps,
- isAppleSilicon,
isAudioModel,
isDatasetAudio,
isDatasetImage,
@@ -55,7 +53,6 @@ export function getDatasetStreamingBlockers({
datasetSource: DatasetSource;
datasetSplit: string | null;
evalSteps: number;
- isAppleSilicon: boolean;
isAudioModel: boolean;
isDatasetAudio: boolean;
isDatasetImage: boolean | null;
@@ -98,8 +95,5 @@ export function getDatasetStreamingBlockers({
if (isDatasetAudio) {
blockers.push("audioDataset");
}
- if (isAppleSilicon) {
- blockers.push("appleSilicon");
- }
return blockers;
}
diff --git a/studio/frontend/src/features/studio/wizard/training-wizard.tsx b/studio/frontend/src/features/studio/wizard/training-wizard.tsx
index 29dd7055249..87d2c83ccd3 100644
--- a/studio/frontend/src/features/studio/wizard/training-wizard.tsx
+++ b/studio/frontend/src/features/studio/wizard/training-wizard.tsx
@@ -16,7 +16,6 @@ import {
TooltipContent,
TooltipTrigger,
} from "@/components/ui/tooltip";
-import { usePlatformStore } from "@/config/env";
import { HfTokenIndicator } from "@/features/hub";
import {
DECISION_MODEL_LABELS,
@@ -27,7 +26,6 @@ import {
type DecisionCheckpoint,
TRAINING_METHOD_META,
TRAINING_METHOD_ORDER,
- isTrainingMethodSupportedOnDevice,
useTrainingConfigStore,
} from "@/features/training";
import { useT } from "@/i18n";
@@ -106,7 +104,6 @@ function TrainingMethodSelect() {
s.modelType === "decision" &&
(s.decisionLayout === "clef" || s.decisionLayout === "llm"),
);
- const deviceType = usePlatformStore((state) => state.deviceType);
const activeMeta = TRAINING_METHOD_META[trainingMethod];
const activeLabel = activeMeta ? t(activeMeta.labelKey) : trainingMethod;
return (
@@ -150,14 +147,10 @@ function TrainingMethodSelect() {
(decisionTakesQlora && method === "qlora"),
).map((method) => {
const meta = TRAINING_METHOD_META[method];
- const unsupportedOnMlx = !isTrainingMethodSupportedOnDevice(
- method,
- deviceType,
- );
return (
<Tooltip key={method} delayDuration={300}>
<TooltipTrigger asChild={true} disableClickToggle={true}>
- <SelectItem value={method} disabled={unsupportedOnMlx}>
+ <SelectItem value={method}>
<span className="flex items-center gap-2">
<span
aria-hidden="true"
@@ -167,11 +160,6 @@ function TrainingMethodSelect() {
)}
/>
{t(meta.labelKey)}
- {unsupportedOnMlx && (
- <span className="text-ui-10 text-muted-foreground">
- · {t("studio.params.notSupportedAppleSilicon")}
- </span>
- )}
</span>
</SelectItem>
</TooltipTrigger>
@@ -180,9 +168,7 @@ function TrainingMethodSelect() {
sideOffset={10}
className="max-w-[calc(220px*var(--ui-space-scale,1))] text-ui-11p5 leading-snug"
>
- {unsupportedOnMlx
- ? t("studio.params.notSupportedAppleSilicon")
- : t(meta.hintKey)}
+ {t(meta.hintKey)}
</TooltipContent>
</Tooltip>
);
diff --git a/studio/frontend/src/features/training/api/mappers.ts b/studio/frontend/src/features/training/api/mappers.ts
index 2c405a2c3da..44792148e99 100644
--- a/studio/frontend/src/features/training/api/mappers.ts
+++ b/studio/frontend/src/features/training/api/mappers.ts
@@ -55,18 +55,24 @@ export function trainingLoadsIn4Bit(
}
/** Whether a run can offload layers: LoRA on the main trainer (embedding and decision models
- * train elsewhere, and the Whisper / codec audio paths have no decoder stack to stream) with
- * gradient checkpointing on, which swapped layers need. */
+ * train elsewhere, and the Whisper / codec audio paths have no decoder stack to stream; an
+ * audio-input vision model such as Gemma 3n/4 does) with gradient checkpointing on, which
+ * swapped layers need. */
export function offloadSupported(
config: Pick<
TrainingConfigState,
- "trainingMethod" | "isEmbeddingModel" | "isAudioModel" | "modelType" | "gradientCheckpointing"
+ | "trainingMethod"
+ | "isEmbeddingModel"
+ | "isAudioModel"
+ | "isVisionModel"
+ | "modelType"
+ | "gradientCheckpointing"
>,
): boolean {
return (
config.trainingMethod !== "full" &&
!config.isEmbeddingModel &&
- !config.isAudioModel &&
+ (!config.isAudioModel || config.isVisionModel) &&
config.modelType !== "embeddings" &&
config.modelType !== "decision" &&
config.gradientCheckpointing !== "none"
@@ -93,6 +99,7 @@ export function offloadPayload(
| "trainingMethod"
| "isEmbeddingModel"
| "isAudioModel"
+ | "isVisionModel"
| "modelType"
| "gradientCheckpointing"
| "offloadLayers"
diff --git a/studio/frontend/src/features/training/index.ts b/studio/frontend/src/features/training/index.ts
index 549ea047cd8..781b9b7bf0f 100644
--- a/studio/frontend/src/features/training/index.ts
+++ b/studio/frontend/src/features/training/index.ts
@@ -22,7 +22,6 @@ export {
getTrainingMethodLabel,
isRawTextDatasetFormat,
isTrainingLoraVariantSupportedOnDevice,
- isTrainingMethodSupportedOnDevice,
isTrainingModelTypeSupportedOnDevice,
parseBackendTrainingMethod,
} from "./lib/training-methods";
diff --git a/studio/frontend/src/features/training/lib/training-methods.ts b/studio/frontend/src/features/training/lib/training-methods.ts
index 6272246daaa..c3965d72fd4 100644
--- a/studio/frontend/src/features/training/lib/training-methods.ts
+++ b/studio/frontend/src/features/training/lib/training-methods.ts
@@ -37,13 +37,6 @@ export function getTrainingMethodLabel(
return TRAINING_METHOD_LABELS.full;
}
-export function isTrainingMethodSupportedOnDevice(
- trainingMethod: TrainingMethod,
- deviceType?: string,
-): boolean {
- return deviceType !== "mac" || trainingMethod !== "cpt";
-}
-
export function isTrainingModelTypeSupportedOnDevice(
modelType: ModelType | null,
deviceType?: string,
diff --git a/studio/frontend/src/features/training/lib/validation.ts b/studio/frontend/src/features/training/lib/validation.ts
index 6238b39b5b9..133dd29b83f 100644
--- a/studio/frontend/src/features/training/lib/validation.ts
+++ b/studio/frontend/src/features/training/lib/validation.ts
@@ -14,7 +14,6 @@ import { isLocalTrainingModelSelection } from "./model-selection";
import { isUntrainableModelFormat } from "./model-support";
import {
isTrainingLoraVariantSupportedOnDevice,
- isTrainingMethodSupportedOnDevice,
isTrainingModelTypeSupportedOnDevice,
} from "./training-methods";
@@ -26,9 +25,6 @@ function isTrainingConfigSupportedOnDevice(
config: TrainingConfigState,
deviceType?: string,
): boolean {
- if (!isTrainingMethodSupportedOnDevice(config.trainingMethod, deviceType)) {
- return false;
- }
if (!isTrainingModelTypeSupportedOnDevice(config.modelType, deviceType)) {
return false;
}
@@ -44,7 +40,12 @@ function isTrainingConfigSupportedOnDevice(
if (deviceType !== "mac") {
return true;
}
- return !(config.isEmbeddingModel || hasChatAudioRows(config, deviceType));
+ // MLX trains audio only as input to an audio-capable vision-language model (Gemma 3n/4).
+ const audioVlm = config.isVisionModel && config.isAudioModel;
+ return !(
+ config.isEmbeddingModel ||
+ (hasChatAudioRows(config, deviceType) && !audioVlm)
+ );
}
/** A Mac Clef (or Clef-head LLM) trainer reads its own audio rows; elsewhere, and for Laya, they train as text. */
@@ -195,7 +196,7 @@ export function validateTrainingConfig(
return { ok: false, errorKey: "studio.modelPicker.reasonGguf" };
}
// Adapter format is valid when continuing LoRA / CPT from a saved adapter (#13140).
- // Decision, embedding and Apple Silicon runs build a fresh adapter, so they need a base.
+ // Decision and embedding runs build a fresh adapter, so they need a base.
// Only a saved local adapter folder is continued; a Hub adapter repo is not yet.
if (
!isAdapterMethod(config.trainingMethod) ||
@@ -205,8 +206,7 @@ export function validateTrainingConfig(
localPath: config.modelLocalPath,
}) ||
config.modelType === "decision" ||
- config.isEmbeddingModel ||
- deviceType === "mac"
+ config.isEmbeddingModel
) {
return { ok: false, errorKey: "studio.modelPicker.reasonAdapter" };
}
diff --git a/studio/frontend/src/i18n/locales/ar.ts b/studio/frontend/src/i18n/locales/ar.ts
index df0974b6c58..248fe8a259a 100644
--- a/studio/frontend/src/i18n/locales/ar.ts
+++ b/studio/frontend/src/i18n/locales/ar.ts
@@ -3930,8 +3930,6 @@ export const ar = {
"يبدو أن مجموعة البيانات هذه تحتوي على صور، ولا يمكن بثها.",
audioDataset:
"يبدو أن مجموعة البيانات هذه تحتوي على صوت، ولا يمكن بثها.",
- appleSilicon:
- "البث غير مدعوم على Apple Silicon (MLX) حتى الآن.",
},
options: {
trainOnCompletions: "إكمالات المساعد فقط",
diff --git a/studio/frontend/src/i18n/locales/cs.ts b/studio/frontend/src/i18n/locales/cs.ts
index 23ca5ffc841..ec0d25700db 100644
--- a/studio/frontend/src/i18n/locales/cs.ts
+++ b/studio/frontend/src/i18n/locales/cs.ts
@@ -4252,8 +4252,6 @@ export const cs = {
"Tato datová sada vypadá jako obrázky, které nelze streamovat.",
audioDataset:
"Tato datová sada vypadá jako zvuk, který nelze streamovat.",
- appleSilicon:
- "Streamování zatím není na Apple Silicon (MLX) podporováno.",
},
options: {
trainOnCompletions: "pouze odpovědi asistenta",
diff --git a/studio/frontend/src/i18n/locales/de.ts b/studio/frontend/src/i18n/locales/de.ts
index 6c2068c2357..c3856141f25 100644
--- a/studio/frontend/src/i18n/locales/de.ts
+++ b/studio/frontend/src/i18n/locales/de.ts
@@ -3986,8 +3986,6 @@ export const de = {
"Dieser Datensatz scheint Bilder zu enthalten, die nicht gestreamt werden können.",
audioDataset:
"Dieser Datensatz scheint Audio zu enthalten, das nicht gestreamt werden kann.",
- appleSilicon:
- "Streaming wird auf Apple Silicon (MLX) noch nicht unterstützt.",
},
options: {
trainOnCompletions: "nur Assistenten-Antworten",
diff --git a/studio/frontend/src/i18n/locales/en.ts b/studio/frontend/src/i18n/locales/en.ts
index f8049f1849f..f779b000874 100644
--- a/studio/frontend/src/i18n/locales/en.ts
+++ b/studio/frontend/src/i18n/locales/en.ts
@@ -3947,7 +3947,6 @@ export const en = {
"Embedding models don't support streaming (training needs the full dataset).",
imageDataset: "This dataset looks like images, which can't stream.",
audioDataset: "This dataset looks like audio, which can't stream.",
- appleSilicon: "Streaming isn't supported on Apple Silicon (MLX) yet.",
},
options: {
trainOnCompletions: "assistant completions only",
diff --git a/studio/frontend/src/i18n/locales/es-419.ts b/studio/frontend/src/i18n/locales/es-419.ts
index e1794ce16b9..e14742e2339 100644
--- a/studio/frontend/src/i18n/locales/es-419.ts
+++ b/studio/frontend/src/i18n/locales/es-419.ts
@@ -4301,8 +4301,6 @@ export const es419 = {
"Este conjunto de datos parece contener imágenes, que no se pueden transmitir.",
audioDataset:
"Este conjunto de datos parece contener audio, que no se puede transmitir.",
- appleSilicon:
- "El streaming todavía no es compatible con Apple Silicon (MLX).",
},
options: {
trainOnCompletions: "solo respuestas del asistente",
diff --git a/studio/frontend/src/i18n/locales/es.ts b/studio/frontend/src/i18n/locales/es.ts
index 77ae7eb8ded..3d18437718d 100644
--- a/studio/frontend/src/i18n/locales/es.ts
+++ b/studio/frontend/src/i18n/locales/es.ts
@@ -3981,8 +3981,6 @@ export const es = {
"Este conjunto de datos parece contener imágenes, que no se pueden transmitir.",
audioDataset:
"Este conjunto de datos parece contener audio, que no se puede transmitir.",
- appleSilicon:
- "El streaming todavía no es compatible con Apple Silicon (MLX).",
},
options: {
trainOnCompletions: "solo respuestas del asistente",
diff --git a/studio/frontend/src/i18n/locales/fa.ts b/studio/frontend/src/i18n/locales/fa.ts
index bdcd9367b40..8be5dfa3ff0 100644
--- a/studio/frontend/src/i18n/locales/fa.ts
+++ b/studio/frontend/src/i18n/locales/fa.ts
@@ -4217,8 +4217,6 @@ export const fa = {
"به نظر می‌رسد این مجموعه‌داده شامل تصویر است که قابل جریان‌دهی نیست.",
audioDataset:
"به نظر می‌رسد این مجموعه‌داده شامل صدا است که قابل جریان‌دهی نیست.",
- appleSilicon:
- "جریان‌دهی هنوز روی Apple Silicon (MLX) پشتیبانی نمی‌شود.",
},
options: {
trainOnCompletions: "فقط تکمیل‌های دستیار",
diff --git a/studio/frontend/src/i18n/locales/fr.ts b/studio/frontend/src/i18n/locales/fr.ts
index 5fbc8d3b793..9a80562fb2b 100644
--- a/studio/frontend/src/i18n/locales/fr.ts
+++ b/studio/frontend/src/i18n/locales/fr.ts
@@ -3993,8 +3993,6 @@ export const fr = {
"Ce jeu de données semble contenir des images, ce qui empêche son utilisation en streaming.",
audioDataset:
"Ce jeu de données semble contenir de l'audio, ce qui empêche son utilisation en streaming.",
- appleSilicon:
- "Le streaming n'est pas encore pris en charge sur Apple Silicon (MLX).",
},
options: {
trainOnCompletions: "réponses de l'assistant uniquement",
diff --git a/studio/frontend/src/i18n/locales/he.ts b/studio/frontend/src/i18n/locales/he.ts
index e3527b6567a..a9ef057aba1 100644
--- a/studio/frontend/src/i18n/locales/he.ts
+++ b/studio/frontend/src/i18n/locales/he.ts
@@ -4006,7 +4006,6 @@ export const he = {
imageDataset:
"מערך נתונים זה נראה כמכיל תמונות, שאינן ניתנות להזרמה.",
audioDataset: "מערך נתונים זה נראה כמכיל אודיו, שאינו ניתן להזרמה.",
- appleSilicon: "הזרמה אינה נתמכת עדיין ב-Apple Silicon (MLX).",
},
options: {
trainOnCompletions: "השלמות עוזר בלבד",
diff --git a/studio/frontend/src/i18n/locales/hi.ts b/studio/frontend/src/i18n/locales/hi.ts
index 9e9aa19aec6..4729396f598 100644
--- a/studio/frontend/src/i18n/locales/hi.ts
+++ b/studio/frontend/src/i18n/locales/hi.ts
@@ -3941,8 +3941,6 @@ export const hi = {
"यह डेटासेट इमेज वाला लगता है, जिसे स्ट्रीम नहीं किया जा सकता।",
audioDataset:
"यह डेटासेट ऑडियो वाला लगता है, जिसे स्ट्रीम नहीं किया जा सकता।",
- appleSilicon:
- "Apple Silicon (MLX) पर अभी स्ट्रीमिंग समर्थित नहीं है।",
},
options: {
trainOnCompletions: "केवल असिस्टेंट completions",
diff --git a/studio/frontend/src/i18n/locales/id.ts b/studio/frontend/src/i18n/locales/id.ts
index 80a4cc0d5e2..d34301cd514 100644
--- a/studio/frontend/src/i18n/locales/id.ts
+++ b/studio/frontend/src/i18n/locales/id.ts
@@ -4229,7 +4229,6 @@ export const id = {
"Dataset ini tampaknya berupa gambar, yang tidak dapat di-streaming.",
audioDataset:
"Dataset ini tampaknya berupa audio, yang tidak dapat di-streaming.",
- appleSilicon: "Streaming belum didukung di Apple Silicon (MLX).",
},
options: {
trainOnCompletions: "hanya completion asisten",
diff --git a/studio/frontend/src/i18n/locales/it.ts b/studio/frontend/src/i18n/locales/it.ts
index 0266b6a0abe..560e6f4b7a6 100644
--- a/studio/frontend/src/i18n/locales/it.ts
+++ b/studio/frontend/src/i18n/locales/it.ts
@@ -4057,8 +4057,6 @@ export const it = {
"Questo dataset sembra contenere immagini, che non possono essere trasmesse in streaming.",
audioDataset:
"Questo dataset sembra contenere audio, che non può essere trasmesso in streaming.",
- appleSilicon:
- "Lo streaming non è ancora supportato su Apple Silicon (MLX).",
},
options: {
trainOnCompletions: "solo completamenti dell'assistente",
diff --git a/studio/frontend/src/i18n/locales/ja.ts b/studio/frontend/src/i18n/locales/ja.ts
index 475666685f1..70f0180fdf1 100644
--- a/studio/frontend/src/i18n/locales/ja.ts
+++ b/studio/frontend/src/i18n/locales/ja.ts
@@ -3901,8 +3901,6 @@ export const ja = {
"このデータセットには画像が含まれているように見えるため、ストリーミングできません。",
audioDataset:
"このデータセットには音声が含まれているように見えるため、ストリーミングできません。",
- appleSilicon:
- "Apple Silicon (MLX) ではストリーミングをまだサポートしていません。",
},
options: {
trainOnCompletions: "アシスタントの応答のみ学習",
diff --git a/studio/frontend/src/i18n/locales/ko.ts b/studio/frontend/src/i18n/locales/ko.ts
index 1776fa86794..1b7ec0d7abc 100644
--- a/studio/frontend/src/i18n/locales/ko.ts
+++ b/studio/frontend/src/i18n/locales/ko.ts
@@ -3921,8 +3921,6 @@ export const ko = {
"이 데이터셋에는 이미지가 포함된 것으로 보여 스트리밍할 수 없습니다.",
audioDataset:
"이 데이터셋에는 오디오가 포함된 것으로 보여 스트리밍할 수 없습니다.",
- appleSilicon:
- "Apple Silicon(MLX)에서는 아직 스트리밍을 지원하지 않습니다.",
},
options: {
trainOnCompletions: "어시스턴트 응답만",
diff --git a/studio/frontend/src/i18n/locales/nl.ts b/studio/frontend/src/i18n/locales/nl.ts
index 7d6e7b07aac..8992c6452be 100644
--- a/studio/frontend/src/i18n/locales/nl.ts
+++ b/studio/frontend/src/i18n/locales/nl.ts
@@ -4282,8 +4282,6 @@ export const nl = {
"Deze dataset lijkt uit afbeeldingen te bestaan, die niet kunnen worden gestreamd.",
audioDataset:
"Deze dataset lijkt uit audio te bestaan, die niet kan worden gestreamd.",
- appleSilicon:
- "Streamen wordt nog niet ondersteund op Apple Silicon (MLX).",
},
options: {
trainOnCompletions: "alleen assistentaanvullingen",
diff --git a/studio/frontend/src/i18n/locales/pl.ts b/studio/frontend/src/i18n/locales/pl.ts
index f4f2acfc691..2c06c403faf 100644
--- a/studio/frontend/src/i18n/locales/pl.ts
+++ b/studio/frontend/src/i18n/locales/pl.ts
@@ -4281,8 +4281,6 @@ export const pl = {
"Ten zbiór danych wygląda na obrazy, których nie można strumieniować.",
audioDataset:
"Ten zbiór danych wygląda na audio, którego nie można strumieniować.",
- appleSilicon:
- "Strumieniowanie nie jest jeszcze obsługiwane na Apple Silicon (MLX).",
},
options: {
trainOnCompletions: "tylko odpowiedzi asystenta",
diff --git a/studio/frontend/src/i18n/locales/pt-br.ts b/studio/frontend/src/i18n/locales/pt-br.ts
index 6006e9f7cd0..dbe0f592d9e 100644
--- a/studio/frontend/src/i18n/locales/pt-br.ts
+++ b/studio/frontend/src/i18n/locales/pt-br.ts
@@ -3952,8 +3952,6 @@ export const ptBR = {
"Este dataset parece conter imagens e não pode ser usado em streaming.",
audioDataset:
"Este dataset parece conter áudio e não pode ser usado em streaming.",
- appleSilicon:
- "O streaming ainda não é compatível com Apple Silicon (MLX).",
},
options: {
trainOnCompletions: "apenas respostas do assistente",
diff --git a/studio/frontend/src/i18n/locales/ru.ts b/studio/frontend/src/i18n/locales/ru.ts
index ab0b330d4c1..85545e746c5 100644
--- a/studio/frontend/src/i18n/locales/ru.ts
+++ b/studio/frontend/src/i18n/locales/ru.ts
@@ -3954,8 +3954,6 @@ export const ru = {
"Похоже, этот датасет содержит изображения, поэтому его нельзя передавать потоком.",
audioDataset:
"Похоже, этот датасет содержит аудио, поэтому его нельзя передавать потоком.",
- appleSilicon:
- "Потоковая передача на Apple Silicon (MLX) пока не поддерживается.",
},
options: {
trainOnCompletions: "только ответы ассистента",
diff --git a/studio/frontend/src/i18n/locales/sv.ts b/studio/frontend/src/i18n/locales/sv.ts
index 3cf60adb56e..bf5001e5be9 100644
--- a/studio/frontend/src/i18n/locales/sv.ts
+++ b/studio/frontend/src/i18n/locales/sv.ts
@@ -4123,7 +4123,6 @@ export const sv = {
"Den här datauppsättningen verkar innehålla bilder, som inte kan strömmas.",
audioDataset:
"Den här datauppsättningen verkar innehålla ljud, som inte kan strömmas.",
- appleSilicon: "Strömning stöds inte på Apple Silicon (MLX) ännu.",
},
options: {
trainOnCompletions: "endast assistentslutföranden",
diff --git a/studio/frontend/src/i18n/locales/th.ts b/studio/frontend/src/i18n/locales/th.ts
index 51793dc6539..8c5005dd3c6 100644
--- a/studio/frontend/src/i18n/locales/th.ts
+++ b/studio/frontend/src/i18n/locales/th.ts
@@ -4044,7 +4044,6 @@ export const th = {
"โมเดล embedding ไม่รองรับการสตรีม (การเทรนต้องใช้ชุดข้อมูลทั้งหมด)",
imageDataset: "ชุดข้อมูลนี้ดูเหมือนเป็นรูปภาพ ซึ่งสตรีมไม่ได้",
audioDataset: "ชุดข้อมูลนี้ดูเหมือนเป็นเสียง ซึ่งสตรีมไม่ได้",
- appleSilicon: "ยังไม่รองรับการสตรีมบน Apple Silicon (MLX)",
},
options: {
trainOnCompletions: "เฉพาะคำตอบของผู้ช่วย",
diff --git a/studio/frontend/src/i18n/locales/tr.ts b/studio/frontend/src/i18n/locales/tr.ts
index c80377c68c8..5db1d72c20d 100644
--- a/studio/frontend/src/i18n/locales/tr.ts
+++ b/studio/frontend/src/i18n/locales/tr.ts
@@ -4229,8 +4229,6 @@ export const tr = {
"Bu veri kümesi görüntülerden oluşuyor gibi görünüyor; görüntüler akış olarak alınamaz.",
audioDataset:
"Bu veri kümesi seslerden oluşuyor gibi görünüyor; sesler akış olarak alınamaz.",
- appleSilicon:
- "Akış henüz Apple Silicon (MLX) üzerinde desteklenmiyor.",
},
options: {
trainOnCompletions: "yalnızca asistan yanıtları",
diff --git a/studio/frontend/src/i18n/locales/uk.ts b/studio/frontend/src/i18n/locales/uk.ts
index 761ffbd6339..fd6b77cd1d3 100644
--- a/studio/frontend/src/i18n/locales/uk.ts
+++ b/studio/frontend/src/i18n/locales/uk.ts
@@ -4270,8 +4270,6 @@ export const uk = {
"Схоже, цей набір даних містить зображення, які не можна передавати потоком.",
audioDataset:
"Схоже, цей набір даних містить аудіо, яке не можна передавати потоком.",
- appleSilicon:
- "Потокове передавання ще не підтримується на Apple Silicon (MLX).",
},
options: {
trainOnCompletions: "лише відповіді асистента",
diff --git a/studio/frontend/src/i18n/locales/vi.ts b/studio/frontend/src/i18n/locales/vi.ts
index d246362b087..51638f8ce62 100644
--- a/studio/frontend/src/i18n/locales/vi.ts
+++ b/studio/frontend/src/i18n/locales/vi.ts
@@ -4231,8 +4231,6 @@ export const vi = {
"Tập dữ liệu này có vẻ là hình ảnh, loại không thể truyền trực tuyến.",
audioDataset:
"Tập dữ liệu này có vẻ là âm thanh, loại không thể truyền trực tuyến.",
- appleSilicon:
- "Chưa hỗ trợ truyền trực tuyến trên Apple Silicon (MLX).",
},
options: {
trainOnCompletions: "chỉ phần trả lời của trợ lý",
diff --git a/studio/frontend/src/i18n/locales/zh-CN.ts b/studio/frontend/src/i18n/locales/zh-CN.ts
index bd35721fa46..d1adae3c611 100644
--- a/studio/frontend/src/i18n/locales/zh-CN.ts
+++ b/studio/frontend/src/i18n/locales/zh-CN.ts
@@ -3854,8 +3854,6 @@ export const zhCN = {
"Embedding 模型不支持流式传输(训练需要完整数据集)。",
imageDataset: "此数据集似乎包含图像,无法进行流式传输。",
audioDataset: "此数据集似乎包含音频,无法进行流式传输。",
- appleSilicon:
- "Apple Silicon(MLX)暂不支持流式传输。",
},
options: {
trainOnCompletions: "仅助手回复",
diff --git a/studio/frontend/src/i18n/locales/zh-TW.ts b/studio/frontend/src/i18n/locales/zh-TW.ts
index 3f07c381963..891fa4f4be1 100644
--- a/studio/frontend/src/i18n/locales/zh-TW.ts
+++ b/studio/frontend/src/i18n/locales/zh-TW.ts
@@ -4013,7 +4013,6 @@ export const zhTW = {
embeddingModel: "嵌入模型不支援串流(訓練需要完整的資料集)。",
imageDataset: "此資料集看起來是圖片,無法串流。",
audioDataset: "此資料集看起來是音訊,無法串流。",
- appleSilicon: "Apple Silicon(MLX)目前尚不支援串流。",
},
options: {
trainOnCompletions: "僅訓練助理回覆",
diff --git a/studio/frontend/tests/training-start-payload-offload.test.ts b/studio/frontend/tests/training-start-payload-offload.test.ts
index 142df891609..e3111052cd4 100644
--- a/studio/frontend/tests/training-start-payload-offload.test.ts
+++ b/studio/frontend/tests/training-start-payload-offload.test.ts
@@ -123,6 +123,8 @@ test("offload goes out off wherever the run cannot offload", () => {
assert.equal(offloadPayload({ ...on, ...blocked }).offload_vram_gb, null);
}
assert.equal(offloadSupported(on), true);
+ // Gemma 3n/4 take audio input on a decoder that streams like any other.
+ assert.equal(offloadSupported({ ...on, isAudioModel: true, isVisionModel: true }), true);
});
test("a count above the backend limit is clamped instead of failing the start", () => {
diff --git a/studio/frontend/tests/training-validation.test.ts b/studio/frontend/tests/training-validation.test.ts
index 8f62908dbbb..d771956573b 100644
--- a/studio/frontend/tests/training-validation.test.ts
+++ b/studio/frontend/tests/training-validation.test.ts
@@ -203,13 +203,6 @@ test("training validation rejects Hub IDs that backend preflight rejects", () =>
});
test("training validation rejects MLX-incompatible training modes", () => {
- assert.deepEqual(
- validateTrainingConfig({ ...validConfig, trainingMethod: "cpt" }, "mac"),
- {
- ok: false,
- errorKey: "studio.params.notSupportedAppleSilicon",
- },
- );
assert.deepEqual(
validateTrainingConfig(
{ ...validConfig, modelType: "embeddings", isEmbeddingModel: true },
@@ -222,7 +215,7 @@ test("training validation rejects MLX-incompatible training modes", () => {
);
});
-test("training validation rejects audio training on MLX", () => {
+test("training validation rejects non-VLM audio training on MLX", () => {
assert.deepEqual(
validateTrainingConfig(
{
@@ -247,6 +240,22 @@ test("training validation rejects audio training on MLX", () => {
);
});
+test("training validation allows audio-capable vision models on MLX with audio data", () => {
+ assert.deepEqual(
+ validateTrainingConfig(
+ {
+ ...validConfig,
+ modelType: "vision",
+ isVisionModel: true,
+ isAudioModel: true,
+ isDatasetAudio: true,
+ },
+ "mac",
+ ),
+ { ok: true, errorKey: null },
+ );
+});
+
test("training validation allows audio-capable vision models on MLX with image data", () => {
assert.deepEqual(
validateTrainingConfig(
@@ -290,18 +299,18 @@ test("training validation accepts DoRA on MLX under an adapter method", () => {
{ ...validConfig, trainingMethod, loraVariant: "dora" },
"mac",
),
- trainingMethod === "cpt"
- ? { ok: false, errorKey: "studio.params.notSupportedAppleSilicon" }
- : { ok: true, errorKey: null },
+ { ok: true, errorKey: null },
);
}
});
-test("training validation keeps CPT and embedding training available off MLX", () => {
- assert.deepEqual(
- validateTrainingConfig({ ...validConfig, trainingMethod: "cpt" }, "linux"),
- { ok: true, errorKey: null },
- );
+test("training validation keeps CPT on every device and embedding training off MLX", () => {
+ for (const deviceType of ["linux", "mac"]) {
+ assert.deepEqual(
+ validateTrainingConfig({ ...validConfig, trainingMethod: "cpt" }, deviceType),
+ { ok: true, errorKey: null },
+ );
+ }
assert.deepEqual(
validateTrainingConfig(
{ ...validConfig, modelType: "embeddings", isEmbeddingModel: true },
@@ -358,6 +367,12 @@ test("training validation lets LoRA and CPT continue from a saved adapter (#1314
{ ok: true, errorKey: null },
);
}
+ for (const trainingMethod of ["lora", "qlora", "cpt"] as const) {
+ assert.deepEqual(
+ validateTrainingConfig({ ...adapter, trainingMethod } as never, "mac"),
+ { ok: true, errorKey: null },
+ );
+ }
});
test("training validation refuses an adapter where a fresh adapter would be built", () => {
@@ -388,7 +403,6 @@ test("training validation refuses an adapter where a fresh adapter would be buil
),
refused,
);
- assert.deepEqual(validateTrainingConfig(adapter as never, "mac"), refused);
});
test("training validation refuses continuing a Hub adapter repo", () => {
diff --git a/tests/studio/test_spark_tts_bicodec_handoff.py b/tests/studio/test_spark_tts_bicodec_handoff.py
index 6f6bb3ee7fd..7c6df72131d 100644
--- a/tests/studio/test_spark_tts_bicodec_handoff.py
+++ b/tests/studio/test_spark_tts_bicodec_handoff.py
@@ -75,6 +75,11 @@ def _run(
"_tokenizer_is_empty": lambda tokenizer: False,
"ensure_spark_tts_source": lambda root: "<source>",
"import_sparktts_module": import_module,
+ "resolve_audio_columns": lambda *args: {
+ "audio_col": "audio",
+ "text_col": "text",
+ "speaker_col": None,
+ },
}
exec(compile(ast.Module(body = nodes, type_ignores = []), str(_TRAINER), "exec"), ns)
owner = SimpleNamespace(
@@ -84,11 +89,6 @@ def _run(
# pre_detect also classifies text-diffusion LMs; a Spark-TTS repo is never one.
_detect_diffusion_lm = lambda *args: None,
diffusion_lm_profile = None,
- _resolve_audio_columns = lambda *args: {
- "audio_col": "audio",
- "text_col": "text",
- "speaker_col": None,
- },
)
owner.resolve_spark_tts_repo = MethodType(ns["resolve_spark_tts_repo"], owner)
dataset = SimpleNamespace(column_names = ["audio", "text"])