Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: price the int8 ConvRot text encoder on Apple Silicon when the dense one is still cached (#13306)

Daniel Hanc860f291cef1
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index f8917c8fbb7..e7199498c24 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -5675,8 +5675,10 @@ class DiffusionBackend:
Uncached checkpoints are priced from the family table (``exact=False``). Never raises."""
try:
from .diffusion_te_prequant import (
+ TE_INT8_CONVROT_BUDGET_SCALE,
TE_PREQUANT_BUDGET_SCALE,
TE_PREQUANT_COMPONENTS,
+ int8_convrot_te_runs_on,
te_candidate_filenames,
te_candidate_is_readable,
te_prequant_sources_for_base,
@@ -5758,7 +5760,13 @@ class DiffusionBackend:
if table is None:
return None
share = table_priced / max(1, len(sources))
- total += int(table[1] * (1000.0**3) * TE_PREQUANT_BUDGET_SCALE * share)
+ # MPS only takes the int8 ConvRot file, smaller than the fp8 one.
+ scale = (
+ TE_INT8_CONVROT_BUDGET_SCALE
+ if int8_convrot_te_runs_on(target)
+ else TE_PREQUANT_BUDGET_SCALE
+ )
+ total += int(table[1] * (1000.0**3) * scale * share)
if total <= 0:
return None
return max(1, total // (1024 * 1024)), tuple(sources), uncached == 0
@@ -9595,8 +9603,12 @@ class DiffusionBackend:
base_local_dir,
)
) // (1024 * 1024)
- # Never below scanned dense shards: a bad checkpoint falls back to opening them.
- precast_mib = max(int(precast_mib), scanned_te)
+ from .diffusion_te_prequant import int8_convrot_te_runs_on
+
+ # Never below scanned dense shards: a bad checkpoint falls back to opening them. Not on MPS, where
+ # that failure raises instead (_te_prequant_pipe_kwargs, #11792).
+ if not int8_convrot_te_runs_on(target):
+ precast_mib = max(int(precast_mib), scanned_te)
text_encoder_mib = max(0, int(text_encoder_mib or 0) - scanned_te) + int(
precast_mib
)
diff --git a/studio/backend/core/inference/diffusion_te_prequant.py b/studio/backend/core/inference/diffusion_te_prequant.py
index 2bb81d7a337..b7f63ce38f7 100644
--- a/studio/backend/core/inference/diffusion_te_prequant.py
+++ b/studio/backend/core/inference/diffusion_te_prequant.py
@@ -797,6 +797,7 @@ def _te_prequant_pipe_kwargs(
still fetch). A pre-cast encoder that fails to load with no dense weights there raises: the plan left
those shards out of the prefetch, so assembly would die on a missing shard instead."""
unavailable: list[str] = []
+ unbudgeted: list[str] = []
failures: list = []
cause = ""
try:
@@ -861,14 +862,24 @@ def _te_prequant_pipe_kwargs(
)
if encoder is not None:
injected[component] = encoder
+ elif int8_convrot_te_runs_on(target):
+ # The MPS plan priced this checkpoint, not the dense shards beside it.
+ unbudgeted.append(component)
+ cause = f" ({failures[-1]})" if failures else cause
elif dense_source and not _hosted_component_cached(dense_source, component, "", ""):
unavailable.append(component)
cause = f" ({failures[-1]})" if failures else cause
- if not unavailable:
+ if not unavailable and not unbudgeted:
return injected
except Exception as exc: # noqa: BLE001 - injection is an optimisation, never a blocker
_warn(logger, "pipe_kwargs", exc)
return {}
+ if unbudgeted:
+ raise RuntimeError(
+ f"The pre-quantized {', '.join(unbudgeted)} for {base} could not be loaded{cause}. The "
+ "memory check sized this load for it, so the larger dense encoder is not loaded in its "
+ "place. Set Text encoder precision to Dense (bf16) to load the dense encoder instead."
+ )
raise RuntimeError(
f"The pre-quantized {', '.join(unavailable)} for {base} could not be loaded{cause}. Its dense "
"weights were not downloaded because the pre-quantized copy replaces them. If memory ran out "
diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.py
index 529dd86ea76..464fc46a6f9 100644
--- a/studio/backend/tests/test_diffusion_backend.py
+++ b/studio/backend/tests/test_diffusion_backend.py
@@ -13663,6 +13663,49 @@ def test_plan_memory_prices_cached_dense_shards_over_a_cached_precast(monkeypatc
assert plan.estimates["model_dense_mib"] == 4350
[email protected]("exact", [True, False])
+def test_mps_prices_the_int8_encoder_over_cached_dense_shards(monkeypatch, tmp_path, exact):
+ # #11792: a 48 GB Mac with 27 GB free that loaded Qwen-Image-2.1 before still holds the dense encoder.
+ from core.inference import diffusion as dmod
+ from core.inference.diffusion_memory import DeviceMemory, unified_memory_shortfall_message
+
+ snapshot = _base_snapshot_with_sizes(
+ tmp_path,
+ monkeypatch,
+ {
+ "text_encoder/model.safetensors": 16722,
+ "vae/diffusion_pytorch_model.safetensors": 650,
+ },
+ )
+ monkeypatch.setattr(
+ dmod,
+ "settled_snapshot_device_memory",
+ lambda t: DeviceMemory("mps", "mps", "unified_memory", 27 * 1024, 48 * 1024),
+ )
+ monkeypatch.setattr(dmod, "estimate_image_runtime_mib", lambda **kw: 8192)
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_precast_text_encoder_mib",
+ staticmethod(lambda *a, **k: (8916, ("text_encoder",), exact)),
+ )
+ target = types.SimpleNamespace(device = "mps", backend = "mps", supports_model_cpu_offload = False)
+ plan = DiffusionBackend()._plan_memory(
+ target,
+ None,
+ "bfl/base",
+ types.SimpleNamespace(name = "qwen-image-2.1"),
+ None,
+ False,
+ kind = "gguf",
+ transformer_resident_override_mib = 4005,
+ base_local_dir = str(snapshot),
+ text_encoder_quant = "int8",
+ )
+ # The int8 file is fetched inside the load, after this check, so an uncached one is priced the same way.
+ assert plan.estimates["text_encoder_dense_mib"] == 8916
+ assert unified_memory_shortfall_message(plan) is None
+
+
def test_cached_dense_shards_keep_the_24g_denoiser_resident(monkeypatch, tmp_path):
from core.inference import diffusion as dmod
from core.inference.diffusion_memory import OFFLOAD_GROUP, DeviceMemory
@@ -13852,6 +13895,27 @@ def test_precast_text_encoder_mib_reads_the_cached_checkpoint(monkeypatch, tmp_p
)
+def test_precast_text_encoder_mib_prices_an_uncached_int8_encoder_on_mps(monkeypatch):
+ # The int8 file is fetched inside the load, after the unified-memory check, so it is uncached there (#11792).
+ import torch
+
+ from core.inference import diffusion_prequant as pq
+ from core.inference.diffusion_auto_policy import family_bf16_components_gb
+ from core.inference.diffusion_families import detect_family
+ from core.inference.diffusion_te_prequant import TE_INT8_CONVROT_BUDGET_SCALE
+
+ fam = detect_family("Qwen/Qwen-Image-2.1")
+ monkeypatch.setattr(pq, "prefer_cached_pickle_twins", lambda repo, names, **k: names)
+ monkeypatch.setattr(pq, "first_cached_as_resolved", lambda *a, **k: None)
+ target = types.SimpleNamespace(device = "mps", dtype = torch.bfloat16, backend = "mps")
+ mib, components, exact = DiffusionBackend._precast_text_encoder_mib(
+ fam, "Qwen/Qwen-Image-2.1", target, "int8"
+ )
+ dense_gb = family_bf16_components_gb(fam, "Qwen/Qwen-Image-2.1")[1]
+ assert (components, exact) == (("text_encoder",), False)
+ assert mib == int(dense_gb * 1000**3 * TE_INT8_CONVROT_BUDGET_SCALE) // (1024 * 1024)
+
+
def test_precast_text_encoder_mib_prices_hidreams_standalone_fourth_encoder(monkeypatch, tmp_path):
from core.inference.diffusion_families import detect_family
from core.inference.diffusion_te_prequant import te_candidate_filenames, te_prequant_sources
diff --git a/studio/backend/tests/test_diffusion_te_release.py b/studio/backend/tests/test_diffusion_te_release.py
index efa78a7b7ba..e25ebc61669 100644
--- a/studio/backend/tests/test_diffusion_te_release.py
+++ b/studio/backend/tests/test_diffusion_te_release.py
@@ -195,6 +195,40 @@ def test_release_env_forces_either_way(monkeypatch, value, expected):
assert te_release.release_wanted(_plan(memory, 1, 1))[0] is expected
[email protected]("device", ["mps", "cuda"])
+def test_a_failed_int8_encoder_on_mps_never_falls_back_to_dense(monkeypatch, device):
+ # The MPS plan priced the int8 file, not the dense shards beside it (#11792).
+ from core.inference import diffusion_precision, media_locality
+
+ monkeypatch.setattr(diffusion_precision, "te_quant_supported", lambda t, m: device == "cuda")
+
+ def _fail(
+ *a,
+ failures = None,
+ **k,
+ ):
+ if failures is not None:
+ failures.append("bad header")
+ return None
+
+ monkeypatch.setattr(te_prequant, "load_prequant_text_encoder", _fail)
+ monkeypatch.setattr(te_prequant, "_held_locally", lambda *a, **k: True)
+ monkeypatch.setattr(media_locality, "_hosted_component_cached", lambda *a, **k: True)
+ call = lambda: te_prequant.te_prequant_pipe_kwargs( # noqa: E731
+ _q21_family(),
+ "Qwen/Qwen-Image-2.1",
+ te_quant_mode = "int8",
+ target = _target(device),
+ dtype = torch.bfloat16,
+ dense_source = "Qwen/Qwen-Image-2.1",
+ )
+ if device == "mps":
+ with pytest.raises(RuntimeError, match = r"bad header.*Dense \(bf16\)"):
+ call()
+ else:
+ assert call() == {}
+
+
def _device() -> str:
if getattr(torch.backends, "mps", None) is not None and torch.backends.mps.is_available():
return "mps"