Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: price the int8 ConvRot text encoder on Apple Silicon when the dense one is still cached (#13306)
Daniel Hanc860f291cef1
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex f8917c8fbb7..e7199498c24 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -5675,8 +5675,10 @@ class DiffusionBackend:Uncached checkpoints are priced from the family table (``exact=False``). Never raises."""try:from .diffusion_te_prequant import (+ TE_INT8_CONVROT_BUDGET_SCALE,TE_PREQUANT_BUDGET_SCALE,TE_PREQUANT_COMPONENTS,+ int8_convrot_te_runs_on,te_candidate_filenames,te_candidate_is_readable,te_prequant_sources_for_base,@@ -5758,7 +5760,13 @@ class DiffusionBackend:if table is None:return Noneshare = table_priced / max(1, len(sources))- total += int(table[1] * (1000.0**3) * TE_PREQUANT_BUDGET_SCALE * share)+ # MPS only takes the int8 ConvRot file, smaller than the fp8 one.+ scale = (+ TE_INT8_CONVROT_BUDGET_SCALE+ if int8_convrot_te_runs_on(target)+ else TE_PREQUANT_BUDGET_SCALE+ )+ total += int(table[1] * (1000.0**3) * scale * share)if total <= 0:return Nonereturn max(1, total // (1024 * 1024)), tuple(sources), uncached == 0@@ -9595,8 +9603,12 @@ class DiffusionBackend:base_local_dir,)) // (1024 * 1024)- # Never below scanned dense shards: a bad checkpoint falls back to opening them.- precast_mib = max(int(precast_mib), scanned_te)+ from .diffusion_te_prequant import int8_convrot_te_runs_on++ # Never below scanned dense shards: a bad checkpoint falls back to opening them. Not on MPS, where+ # that failure raises instead (_te_prequant_pipe_kwargs, #11792).+ if not int8_convrot_te_runs_on(target):+ precast_mib = max(int(precast_mib), scanned_te)text_encoder_mib = max(0, int(text_encoder_mib or 0) - scanned_te) + int(precast_mib)diff --git a/studio/backend/core/inference/diffusion_te_prequant.py b/studio/backend/core/inference/diffusion_te_prequant.pyindex 2bb81d7a337..b7f63ce38f7 100644--- a/studio/backend/core/inference/diffusion_te_prequant.py+++ b/studio/backend/core/inference/diffusion_te_prequant.py@@ -797,6 +797,7 @@ def _te_prequant_pipe_kwargs(still fetch). A pre-cast encoder that fails to load with no dense weights there raises: the plan leftthose shards out of the prefetch, so assembly would die on a missing shard instead."""unavailable: list[str] = []+ unbudgeted: list[str] = []failures: list = []cause = ""try:@@ -861,14 +862,24 @@ def _te_prequant_pipe_kwargs()if encoder is not None:injected[component] = encoder+ elif int8_convrot_te_runs_on(target):+ # The MPS plan priced this checkpoint, not the dense shards beside it.+ unbudgeted.append(component)+ cause = f" ({failures[-1]})" if failures else causeelif dense_source and not _hosted_component_cached(dense_source, component, "", ""):unavailable.append(component)cause = f" ({failures[-1]})" if failures else cause- if not unavailable:+ if not unavailable and not unbudgeted:return injectedexcept Exception as exc: # noqa: BLE001 - injection is an optimisation, never a blocker_warn(logger, "pipe_kwargs", exc)return {}+ if unbudgeted:+ raise RuntimeError(+ f"The pre-quantized {', '.join(unbudgeted)} for {base} could not be loaded{cause}. The "+ "memory check sized this load for it, so the larger dense encoder is not loaded in its "+ "place. Set Text encoder precision to Dense (bf16) to load the dense encoder instead."+ )raise RuntimeError(f"The pre-quantized {', '.join(unavailable)} for {base} could not be loaded{cause}. Its dense ""weights were not downloaded because the pre-quantized copy replaces them. If memory ran out "diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.pyindex 529dd86ea76..464fc46a6f9 100644--- a/studio/backend/tests/test_diffusion_backend.py+++ b/studio/backend/tests/test_diffusion_backend.py@@ -13663,6 +13663,49 @@ def test_plan_memory_prices_cached_dense_shards_over_a_cached_precast(monkeypatcassert plan.estimates["model_dense_mib"] == 4350[email protected]("exact", [True, False])+def test_mps_prices_the_int8_encoder_over_cached_dense_shards(monkeypatch, tmp_path, exact):+ # #11792: a 48 GB Mac with 27 GB free that loaded Qwen-Image-2.1 before still holds the dense encoder.+ from core.inference import diffusion as dmod+ from core.inference.diffusion_memory import DeviceMemory, unified_memory_shortfall_message++ snapshot = _base_snapshot_with_sizes(+ tmp_path,+ monkeypatch,+ {+ "text_encoder/model.safetensors": 16722,+ "vae/diffusion_pytorch_model.safetensors": 650,+ },+ )+ monkeypatch.setattr(+ dmod,+ "settled_snapshot_device_memory",+ lambda t: DeviceMemory("mps", "mps", "unified_memory", 27 * 1024, 48 * 1024),+ )+ monkeypatch.setattr(dmod, "estimate_image_runtime_mib", lambda **kw: 8192)+ monkeypatch.setattr(+ DiffusionBackend,+ "_precast_text_encoder_mib",+ staticmethod(lambda *a, **k: (8916, ("text_encoder",), exact)),+ )+ target = types.SimpleNamespace(device = "mps", backend = "mps", supports_model_cpu_offload = False)+ plan = DiffusionBackend()._plan_memory(+ target,+ None,+ "bfl/base",+ types.SimpleNamespace(name = "qwen-image-2.1"),+ None,+ False,+ kind = "gguf",+ transformer_resident_override_mib = 4005,+ base_local_dir = str(snapshot),+ text_encoder_quant = "int8",+ )+ # The int8 file is fetched inside the load, after this check, so an uncached one is priced the same way.+ assert plan.estimates["text_encoder_dense_mib"] == 8916+ assert unified_memory_shortfall_message(plan) is None++def test_cached_dense_shards_keep_the_24g_denoiser_resident(monkeypatch, tmp_path):from core.inference import diffusion as dmodfrom core.inference.diffusion_memory import OFFLOAD_GROUP, DeviceMemory@@ -13852,6 +13895,27 @@ def test_precast_text_encoder_mib_reads_the_cached_checkpoint(monkeypatch, tmp_p)+def test_precast_text_encoder_mib_prices_an_uncached_int8_encoder_on_mps(monkeypatch):+ # The int8 file is fetched inside the load, after the unified-memory check, so it is uncached there (#11792).+ import torch++ from core.inference import diffusion_prequant as pq+ from core.inference.diffusion_auto_policy import family_bf16_components_gb+ from core.inference.diffusion_families import detect_family+ from core.inference.diffusion_te_prequant import TE_INT8_CONVROT_BUDGET_SCALE++ fam = detect_family("Qwen/Qwen-Image-2.1")+ monkeypatch.setattr(pq, "prefer_cached_pickle_twins", lambda repo, names, **k: names)+ monkeypatch.setattr(pq, "first_cached_as_resolved", lambda *a, **k: None)+ target = types.SimpleNamespace(device = "mps", dtype = torch.bfloat16, backend = "mps")+ mib, components, exact = DiffusionBackend._precast_text_encoder_mib(+ fam, "Qwen/Qwen-Image-2.1", target, "int8"+ )+ dense_gb = family_bf16_components_gb(fam, "Qwen/Qwen-Image-2.1")[1]+ assert (components, exact) == (("text_encoder",), False)+ assert mib == int(dense_gb * 1000**3 * TE_INT8_CONVROT_BUDGET_SCALE) // (1024 * 1024)++def test_precast_text_encoder_mib_prices_hidreams_standalone_fourth_encoder(monkeypatch, tmp_path):from core.inference.diffusion_families import detect_familyfrom core.inference.diffusion_te_prequant import te_candidate_filenames, te_prequant_sourcesdiff --git a/studio/backend/tests/test_diffusion_te_release.py b/studio/backend/tests/test_diffusion_te_release.pyindex efa78a7b7ba..e25ebc61669 100644--- a/studio/backend/tests/test_diffusion_te_release.py+++ b/studio/backend/tests/test_diffusion_te_release.py@@ -195,6 +195,40 @@ def test_release_env_forces_either_way(monkeypatch, value, expected):assert te_release.release_wanted(_plan(memory, 1, 1))[0] is expected[email protected]("device", ["mps", "cuda"])+def test_a_failed_int8_encoder_on_mps_never_falls_back_to_dense(monkeypatch, device):+ # The MPS plan priced the int8 file, not the dense shards beside it (#11792).+ from core.inference import diffusion_precision, media_locality++ monkeypatch.setattr(diffusion_precision, "te_quant_supported", lambda t, m: device == "cuda")++ def _fail(+ *a,+ failures = None,+ **k,+ ):+ if failures is not None:+ failures.append("bad header")+ return None++ monkeypatch.setattr(te_prequant, "load_prequant_text_encoder", _fail)+ monkeypatch.setattr(te_prequant, "_held_locally", lambda *a, **k: True)+ monkeypatch.setattr(media_locality, "_hosted_component_cached", lambda *a, **k: True)+ call = lambda: te_prequant.te_prequant_pipe_kwargs( # noqa: E731+ _q21_family(),+ "Qwen/Qwen-Image-2.1",+ te_quant_mode = "int8",+ target = _target(device),+ dtype = torch.bfloat16,+ dense_source = "Qwen/Qwen-Image-2.1",+ )+ if device == "mps":+ with pytest.raises(RuntimeError, match = r"bad header.*Dense \(bf16\)"):+ call()+ else:+ assert call() == {}++def _device() -> str:if getattr(torch.backends, "mps", None) is not None and torch.backends.mps.is_available():return "mps"