Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

fix(studio): record the bit widths of prequantized MLX models (#11940)

Tom Mathews6f70f901e48f
diff --git a/studio/backend/core/training/provenance.py b/studio/backend/core/training/provenance.py
index 6781c37959e..7b44a79ee16 100644
--- a/studio/backend/core/training/provenance.py
+++ b/studio/backend/core/training/provenance.py
@@ -19,8 +19,10 @@ RESOURCE_PROVENANCE_KEY = "resource_provenance"
_ATTESTED = "attested"
_INCOMPLETE = "incomplete"
_MODEL_LOAD_UNQUANTIZED = "unquantized"
+# `prequantized_{sorted unique widths joined by _}bit`; the {4} case keeps v1 markers valid.
_MODEL_LOAD_PREQUANTIZED_4BIT = "prequantized_4bit"
_MODEL_LOAD_RUNTIME_4BIT = "runtime_4bit"
+_PREQUANTIZED_MODE_RE = re.compile(r"prequantized_(?:1[0-6]|[1-9])(?:_(?:1[0-6]|[1-9])){0,7}bit")
_REASON_RE = re.compile(r"[a-z0-9][a-z0-9_-]{0,63}")
_MODEL_WEIGHT_RE = re.compile(
r"(?:"
@@ -161,37 +163,53 @@ def _normalized_commit(value: Any) -> Optional[str]:
return value
-def _snapshot_declares_quantization(snapshot: Path) -> bool:
+def _prequantized_mode(widths: frozenset[int]) -> str:
+ return "prequantized_" + "_".join(str(width) for width in sorted(widths)) + "bit"
+
+
+def _prequantized_mode_widths(mode: Any) -> Optional[frozenset[int]]:
+ if not isinstance(mode, str) or _PREQUANTIZED_MODE_RE.fullmatch(mode) is None:
+ return None
+ widths = [
+ int(part) for part in mode.removeprefix("prequantized_").removesuffix("bit").split("_")
+ ]
+ # Only the canonical (strictly increasing) spelling `_prequantized_mode` writes attests.
+ if any(widths[index] >= widths[index + 1] for index in range(len(widths) - 1)):
+ return None
+ return frozenset(widths)
+
+
+def _snapshot_quantization_widths(snapshot: Path) -> Optional[frozenset[int]]:
try:
parsed = json.loads((snapshot / "config.json").read_text(encoding = "utf-8"))
except (OSError, ValueError):
- return False
+ return frozenset()
if not isinstance(parsed, dict):
- return False
+ return frozenset()
queue = [parsed.get("quantization_config"), parsed.get("quantization")]
- found_4bit = False
- conflicting_width = False
+ widths: set[int] = set()
while queue:
current = queue.pop()
if isinstance(current, dict):
if current.get("load_in_4bit") is True:
- found_4bit = True
+ widths.add(4)
if current.get("load_in_8bit") is True:
- conflicting_width = True
+ widths.add(8)
for key in ("bits", "nbits", "q_bits"):
width = current.get(key)
if isinstance(width, str) and width.strip().isdigit():
width = int(width.strip())
if isinstance(width, bool) or not isinstance(width, int):
continue
- if width == 4:
- found_4bit = True
- else:
- conflicting_width = True
+ widths.add(width)
queue.extend(current.values())
elif isinstance(current, (list, tuple)):
queue.extend(current)
- return found_4bit and not conflicting_width
+ if not widths:
+ return frozenset()
+ if len(widths) > 8 or any(width < 1 or width > 16 for width in widths):
+ return None
+ return frozenset(widths)
def _resolved_model_snapshot_file(snapshot: Path, path: Path) -> Optional[Path]:
@@ -272,12 +290,7 @@ def _snapshot_has_dataset_data(snapshot: Path) -> bool:
return found_data
-def exact_model_snapshot_path(
- path_value: Any,
- repo_id: Any,
- *,
- require_quantized: bool = False,
-) -> Optional[str]:
+def exact_model_snapshot_path(path_value: Any, repo_id: Any) -> Optional[str]:
repo_id = _normalized_repo_id(repo_id)
if repo_id is None or not isinstance(path_value, str) or not path_value.strip():
return None
@@ -306,17 +319,10 @@ def exact_model_snapshot_path(
return None
if not same_existing_path(resolved, requested) or not _snapshot_has_model_weights(resolved):
return None
- if require_quantized and not _snapshot_declares_quantization(resolved):
- return None
return str(resolved)
-def exact_model_snapshot_for_commit(
- repo_id: Any,
- commit: Any,
- *,
- require_quantized: bool = False,
-) -> Optional[str]:
+def exact_model_snapshot_for_commit(repo_id: Any, commit: Any) -> Optional[str]:
repo_id = _normalized_repo_id(repo_id)
commit = _normalized_commit(commit)
if repo_id is None or commit is None:
@@ -329,7 +335,6 @@ def exact_model_snapshot_for_commit(
resolved = exact_model_snapshot_path(
str(candidate),
repo_id,
- require_quantized = require_quantized,
)
if resolved is not None:
return resolved
@@ -612,10 +617,13 @@ def _loaded_model_refs(model: Any) -> set[tuple[str, str]]:
def _attested_model_load_mode(snapshot: str, model: Any, load_in_4bit: bool) -> Optional[str]:
+ widths = _snapshot_quantization_widths(Path(snapshot))
+ if widths is None:
+ return None
+ if widths:
+ return _prequantized_mode(widths)
if not load_in_4bit:
return _MODEL_LOAD_UNQUANTIZED
- if _snapshot_declares_quantization(Path(snapshot)):
- return _MODEL_LOAD_PREQUANTIZED_4BIT
if _loaded_model_is_4bit(model):
return _MODEL_LOAD_RUNTIME_4BIT
return None
@@ -757,17 +765,24 @@ def normalize_worker_provenance_event(
)
if model_snapshot is not None:
event_load_mode = model_event.get("load_mode")
- snapshot_is_quantized = _snapshot_declares_quantization(Path(model_snapshot))
- if bool(config.get("load_in_4bit")):
- if (
- event_load_mode in (None, _MODEL_LOAD_PREQUANTIZED_4BIT)
- and snapshot_is_quantized
- ):
- model_load_mode = _MODEL_LOAD_PREQUANTIZED_4BIT
- elif event_load_mode == _MODEL_LOAD_RUNTIME_4BIT and not snapshot_is_quantized:
- model_load_mode = _MODEL_LOAD_RUNTIME_4BIT
- elif event_load_mode in (None, _MODEL_LOAD_UNQUANTIZED):
- model_load_mode = _MODEL_LOAD_UNQUANTIZED
+ snapshot_widths = _snapshot_quantization_widths(Path(model_snapshot))
+ load_in_4bit = bool(config.get("load_in_4bit"))
+ event_widths = _prequantized_mode_widths(event_load_mode)
+ if snapshot_widths is not None:
+ if event_widths is not None:
+ if event_widths == snapshot_widths:
+ model_load_mode = _prequantized_mode(event_widths)
+ elif event_load_mode is None:
+ if load_in_4bit and snapshot_widths == frozenset({4}):
+ model_load_mode = _MODEL_LOAD_PREQUANTIZED_4BIT
+ elif not load_in_4bit and not snapshot_widths:
+ model_load_mode = _MODEL_LOAD_UNQUANTIZED
+ elif event_load_mode == _MODEL_LOAD_RUNTIME_4BIT:
+ if load_in_4bit and not snapshot_widths:
+ model_load_mode = _MODEL_LOAD_RUNTIME_4BIT
+ elif event_load_mode == _MODEL_LOAD_UNQUANTIZED:
+ if not load_in_4bit and not snapshot_widths:
+ model_load_mode = _MODEL_LOAD_UNQUANTIZED
if model_load_mode is None:
model_snapshot = None
if model_snapshot is None:
@@ -818,14 +833,16 @@ def validate_exact_model_pin(config: dict[str, Any]) -> str:
if stored_load_mode is None and isinstance(marker, dict):
stored_load_mode = marker.get("model_load_mode")
load_in_4bit = bool(config.get("load_in_4bit"))
- if load_in_4bit:
+ expected_widths = _prequantized_mode_widths(stored_load_mode)
+ if expected_widths is not None:
+ model_load_mode = _prequantized_mode(expected_widths)
+ elif load_in_4bit:
if stored_load_mode is None:
model_load_mode = _MODEL_LOAD_PREQUANTIZED_4BIT
- elif stored_load_mode in {
- _MODEL_LOAD_PREQUANTIZED_4BIT,
- _MODEL_LOAD_RUNTIME_4BIT,
- }:
- model_load_mode = stored_load_mode
+ expected_widths = frozenset({4})
+ elif stored_load_mode == _MODEL_LOAD_RUNTIME_4BIT:
+ model_load_mode = _MODEL_LOAD_RUNTIME_4BIT
+ expected_widths = frozenset()
else:
model_load_mode = None
elif stored_load_mode in (None, _MODEL_LOAD_UNQUANTIZED):
@@ -844,12 +861,12 @@ def validate_exact_model_pin(config: dict[str, Any]) -> str:
model_snapshot = exact_model_snapshot_path(
config.get("model_snapshot_path"),
model_repo_id,
- require_quantized = model_load_mode == _MODEL_LOAD_PREQUANTIZED_4BIT,
)
+ # Legacy `unquantized` markers never checked widths; keep them resumable.
if (
model_snapshot is not None
- and model_load_mode == _MODEL_LOAD_RUNTIME_4BIT
- and _snapshot_declares_quantization(Path(model_snapshot))
+ and model_load_mode != _MODEL_LOAD_UNQUANTIZED
+ and _snapshot_quantization_widths(Path(model_snapshot)) != expected_widths
):
model_snapshot = None
if model_snapshot is None:
diff --git a/studio/backend/tests/test_training_cached_start.py b/studio/backend/tests/test_training_cached_start.py
index 86427b81a76..a4ae0750527 100644
--- a/studio/backend/tests/test_training_cached_start.py
+++ b/studio/backend/tests/test_training_cached_start.py
@@ -1566,14 +1566,29 @@ def test_unscoped_reset_cannot_touch_a_live_run(monkeypatch):
assert backend.reset_training_state() == "active"
-def test_runtime_4bit_resume_reaches_worker_with_source_resource_pins(tmp_path):
- route = _load_route_module("training_route_resume_resource_provenance")
+ ("resume_model_load_mode", "model_config_json"),
+ [
+ pytest.param("runtime_4bit", "{}", id = "runtime_4bit"),
+ pytest.param(
+ "prequantized_8bit",
+ json.dumps({"quantization_config": {"load_in_8bit": True}}),
+ id = "prequantized_8bit",
+ ),
+ ],
+)
+def test_4bit_resume_reaches_worker_with_source_resource_pins(
+ tmp_path, resume_model_load_mode, model_config_json
+):
+ route = _load_route_module(
+ f"training_route_resume_resource_provenance_{resume_model_load_mode}"
+ )
model_root = tmp_path / "models--unsloth--test"
old_model = model_root / "snapshots" / "commit-old"
new_model = model_root / "snapshots" / "commit-new"
for snapshot in (old_model, new_model):
snapshot.mkdir(parents = True)
- (snapshot / "config.json").write_text("{}")
+ (snapshot / "config.json").write_text(model_config_json)
(snapshot / "model.safetensors").write_bytes(b"x")
dataset_root = tmp_path / "datasets--org--dataset"
old_dataset = dataset_root / "snapshots" / "commit-old"
@@ -1599,7 +1614,7 @@ def test_runtime_4bit_resume_reaches_worker_with_source_resource_pins(tmp_path):
"version": 1,
"status": "complete",
"model_status": "attested",
- "model_load_mode": "runtime_4bit",
+ "model_load_mode": resume_model_load_mode,
"dataset_status": "attested",
"reasons": [],
},
@@ -1662,7 +1677,7 @@ def test_runtime_4bit_resume_reaches_worker_with_source_resource_pins(tmp_path):
assert captured["require_exact_resume_resources"] is True
assert captured["require_exact_model_resource"] is True
assert captured["require_exact_dataset_resource"] is True
- assert captured["resume_model_load_mode"] == "runtime_4bit"
+ assert captured["resume_model_load_mode"] == resume_model_load_mode
assert tier_targets == [str(old_model.resolve())]
from core.training.training import (
@@ -1677,7 +1692,7 @@ def test_runtime_4bit_resume_reaches_worker_with_source_resource_pins(tmp_path):
worker_config = _build_training_worker_config(captured)
_apply_cache_pins(worker_config)
- assert worker_config["resume_model_load_mode"] == "runtime_4bit"
+ assert worker_config["resume_model_load_mode"] == resume_model_load_mode
assert worker_config["model_snapshot_path"] == str(old_model.resolve())
diff --git a/studio/backend/tests/test_training_provenance.py b/studio/backend/tests/test_training_provenance.py
index 776e586dcfe..d3dcc0eedbc 100644
--- a/studio/backend/tests/test_training_provenance.py
+++ b/studio/backend/tests/test_training_provenance.py
@@ -23,6 +23,7 @@ from core.training.provenance import (
exact_model_snapshot_path,
normalize_worker_provenance_event,
resource_provenance_allows_resume,
+ validate_exact_model_pin,
)
from hub.utils import dataset_cache, hf_cache_state
@@ -559,18 +560,21 @@ def test_mlx_top_level_quantization_attests_prequantized_snapshot(tmp_path):
assert resource_provenance_allows_resume({**config, **updates}) is True
-def test_mlx_top_level_8bit_quantization_does_not_attest_as_4bit(tmp_path):
+def test_mlx_top_level_8bit_quantization_attests_prequantized_8bit_snapshot(tmp_path):
model_snapshot = _model_snapshot(tmp_path, "org/model", "model-commit")
(model_snapshot / "config.json").write_text(
json.dumps({"quantization": {"bits": 8, "group_size": 64}}),
encoding = "utf-8",
)
config, event = _shared_setup_2(model_snapshot, tmp_path)
+ updates = normalize_worker_provenance_event(event, config)
- assert event["model"]["status"] == "incomplete"
+ assert event["model"]["load_mode"] == "prequantized_8bit"
+ assert updates[RESOURCE_PROVENANCE_KEY]["status"] == "complete"
+ assert resource_provenance_allows_resume({**config, **updates}) is True
-def test_mlx_conflicting_quantization_widths_do_not_attest_as_4bit(tmp_path):
+def test_mlx_mixed_quantization_widths_attest_prequantized_4_8bit_snapshot(tmp_path):
model_snapshot = _model_snapshot(tmp_path, "org/model", "model-commit")
(model_snapshot / "config.json").write_text(
json.dumps(
@@ -585,6 +589,121 @@ def test_mlx_conflicting_quantization_widths_do_not_attest_as_4bit(tmp_path):
encoding = "utf-8",
)
config, event = _shared_setup_2(model_snapshot, tmp_path)
+ updates = normalize_worker_provenance_event(event, config)
+
+ assert event["model"]["load_mode"] == "prequantized_4_8bit"
+ assert resource_provenance_allows_resume({**config, **updates}) is True
+
+
+def test_mlx_top_level_8bit_quantization_attests_regardless_of_the_4bit_toggle(tmp_path):
+ model_snapshot = _model_snapshot(tmp_path, "org/model", "model-commit")
+ (model_snapshot / "config.json").write_text(
+ json.dumps({"quantization": {"bits": 8, "group_size": 64}}),
+ encoding = "utf-8",
+ )
+ dataset = _dataset_snapshot(tmp_path, "org/dataset", "dataset-commit")
+ config = {
+ "model_name": "org/model",
+ "model_snapshot_path": str(model_snapshot),
+ "hf_dataset": "org/dataset",
+ "dataset_snapshot_path": str(dataset),
+ "load_in_4bit": False,
+ }
+ event = build_worker_provenance_event(
+ config,
+ SimpleNamespace(),
+ model_load_target = str(model_snapshot),
+ model_load_in_4bit = False,
+ dataset_loaded_from_exact_snapshot = True,
+ )
+ updates = normalize_worker_provenance_event(event, config)
+
+ assert event["model"]["load_mode"] == "prequantized_8bit"
+ assert resource_provenance_allows_resume({**config, **updates}) is True
+
+
+def test_prequantized_8bit_pin_refuses_resume_once_the_snapshot_declares_4bit(tmp_path):
+ model_snapshot = _model_snapshot(tmp_path, "org/model", "model-commit")
+ (model_snapshot / "config.json").write_text(
+ json.dumps({"quantization": {"bits": 8}}),
+ encoding = "utf-8",
+ )
+ config = {
+ "model_name": "org/model",
+ "model_snapshot_path": str(model_snapshot),
+ "load_in_4bit": True,
+ "resume_model_load_mode": "prequantized_8bit",
+ }
+ assert validate_exact_model_pin(config) == str(model_snapshot.resolve())
+
+ (model_snapshot / "config.json").write_text(
+ json.dumps({"quantization": {"bits": 4}}),
+ encoding = "utf-8",
+ )
+ with pytest.raises(ExactResumeResourcesUnavailable):
+ validate_exact_model_pin(config)
+
+
+def test_legacy_unquantized_marker_resumes_even_if_the_snapshot_now_declares_8bit(tmp_path):
+ model_snapshot = _model_snapshot(tmp_path, "org/model", "model-commit")
+ (model_snapshot / "config.json").write_text(
+ json.dumps({"quantization": {"bits": 8}}),
+ encoding = "utf-8",
+ )
+ config = {
+ "model_name": "org/model",
+ "model_snapshot_path": str(model_snapshot),
+ "load_in_4bit": False,
+ RESOURCE_PROVENANCE_KEY: {
+ "version": 1,
+ "status": "complete",
+ "model_load_mode": "unquantized",
+ },
+ }
+ assert validate_exact_model_pin(config) == str(model_snapshot.resolve())
+
+
+ ("stored_mode", "canonical_mode", "quantization"),
+ [
+ ("prequantized_08bit", "prequantized_8bit", {"bits": 8}),
+ ("prequantized_8_8bit", "prequantized_8bit", {"bits": 8}),
+ (
+ "prequantized_8_4bit",
+ "prequantized_4_8bit",
+ {"bits": 8, "overrides": {"decoder.layers.0": {"bits": 4}}},
+ ),
+ ("prequantized_bit", "prequantized_8bit", {"bits": 8}),
+ ],
+)
+def test_non_canonical_stored_load_mode_refuses_resume(
+ tmp_path, stored_mode, canonical_mode, quantization
+):
+ model_snapshot = _model_snapshot(tmp_path, "org/model", "model-commit")
+ (model_snapshot / "config.json").write_text(
+ json.dumps({"quantization": quantization}),
+ encoding = "utf-8",
+ )
+ config = {
+ "model_name": "org/model",
+ "model_snapshot_path": str(model_snapshot),
+ "load_in_4bit": True,
+ }
+
+ assert validate_exact_model_pin({**config, "resume_model_load_mode": canonical_mode}) == str(
+ model_snapshot.resolve()
+ )
+ with pytest.raises(ExactResumeResourcesUnavailable):
+ validate_exact_model_pin({**config, "resume_model_load_mode": stored_mode})
+
+
+def test_snapshot_declaring_out_of_range_width_does_not_attest(tmp_path):
+ model_snapshot = _model_snapshot(tmp_path, "org/model", "model-commit")
+ (model_snapshot / "config.json").write_text(
+ json.dumps({"quantization": {"bits": 32}}),
+ encoding = "utf-8",
+ )
+ config, event = _shared_setup_2(model_snapshot, tmp_path)
assert event["model"]["status"] == "incomplete"