Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop trading the denoiser's quantisation away to pay for offload (#11558)

Daniel Hanb6ee1739d471
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index f51a1de2358..8f4bdd1c2b8 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -103,6 +103,7 @@ from .diffusion_memory import (
MEMORY_MODE_LOW_VRAM,
ACTIVATION_TILE,
DeviceMemory,
+ OFFLOAD_MODEL,
OFFLOAD_NONE,
OFFLOAD_STREAMING,
apply_memory_plan,
@@ -111,6 +112,7 @@ from .diffusion_memory import (
estimate_image_runtime_mib,
estimate_safetensors_dense_mib,
file_size_mib,
+ largest_streamable_companion_mib,
normalize_memory_mode,
plan_diffusion_memory,
plan_fits_total_capacity,
@@ -2052,16 +2054,19 @@ class DiffusionBackend:
is None
):
reason = explain_unusable_scheme(getattr(fam, "name", None), pinned)
- elif _memory_request_forces_offload(memory_mode, cpu_offload):
+ elif _memory_request_forces_offload(memory_mode, cpu_offload) and (
+ model_kind != "pipeline"
+ or normalize_memory_mode(memory_mode) == MEMORY_MODE_BALANCED
+ ):
# Not a measurement: balanced and low_vram name their policy outright, and the legacy flag forces
- # model offload. Offload hooks move modules with Module.to(), which torchao tensors do not survive, so
- # the loader skips the dense build for any of them, and the strict refusal then landed after the
- # resident model was already gone. The two requests are incompatible on their face.
+ # model offload; refusing late would land after the resident model was already evicted.
requested_memory = normalize_memory_mode(memory_mode) or "cpu_offload"
reason = (
- f"'{requested_memory}' memory places the transformer under CPU offload, and "
- "torchao quantised tensors cannot be moved by the offload hooks, so the dense "
- "quant is skipped"
+ f"'{requested_memory}' memory places the transformer under CPU offload, and a "
+ "GGUF pick only swaps in the quantised dense transformer when it fits resident"
+ if model_kind != "pipeline"
+ else "'balanced' memory streams the transformer through group-offload hooks "
+ "torchao weights do not survive; use low_vram to combine the two"
)
elif not dense_transformer_supported(target):
reason = dense_transformer_unsupported_reason(target)
@@ -6217,11 +6222,7 @@ class DiffusionBackend:
)
# Quantise dense bf16 pipeline denoisers in place. The blocker excludes UNet and pre-quantised
- # pipelines; offloaded plans stay dense because torchao tensors cannot move. The pipeline is still
- # on the CPU here, unlike the GGUF path, which quantises after _assemble_pipe places it. Both
- # orders give bit-identical output: apply_memory_plan's one-shot `pipe.to(placement)` is survived
- # by the subclasses (measured on sm_89, fp8 and int8, max|diff| 0.0); only the per-forward offload
- # hooks are not.
+ # pipelines; offloaded plans stay dense unless whole-module.
if (
pipe is not None
and kind == "pipeline"
@@ -6308,16 +6309,37 @@ class DiffusionBackend:
native_scheme,
plan.offload_policy,
)
- if not keeps_resident and native_scheme is None:
- logger.info(
- "diffusion.transformer_quant: skipped (the memory plan picked '%s' "
- "offload, which moves the transformer via Module.to())",
- plan.offload_policy,
+ # Group offload is WRONG for torchao: its stream cache aliases weights and swap_tensors fails when compiled.
+ quant_budget = int(plan.estimates.get("safe_device_budget_mib") or 0)
+ quant_overhead = int(
+ plan.estimates.get("runtime_headroom_mib") or 0
+ ) + int(plan.estimates.get("base_overhead_mib") or 0)
+ if (
+ not keeps_resident
+ and native_scheme is None
+ and (
+ plan.offload_policy != OFFLOAD_MODEL
+ or estimate is None
+ or estimate.steady_transformer_mib + quant_overhead
+ > quant_budget
+ or (largest_streamable_companion_mib(pipe) or 0) > quant_budget
)
- transformer_quant_decline = (
- f"the memory plan picked '{plan.offload_policy}' offload, which moves "
- "the transformer via Module.to(); torchao quantised tensors reject "
- "that. Pin a resident memory mode to combine the two"
+ ):
+ if plan.offload_policy == OFFLOAD_MODEL:
+ transformer_quant_decline = (
+ "whole-module offload onloads each component whole, and the "
+ "quantised transformer or a text encoder is not known to fit "
+ "the device budget"
+ )
+ else:
+ transformer_quant_decline = (
+ f"'{plan.offload_policy}' offload streams the transformer through "
+ "hooks torchao weights do not survive. Pin low_vram or a resident "
+ "memory mode to combine the two"
+ )
+ logger.info(
+ "diffusion.transformer_quant: skipped (%s)",
+ transformer_quant_decline,
)
elif native_scheme is None and pipeline_quant_uncompilable is not None:
# Resident plan: torchao still needs the compile check skipped above.
@@ -8867,8 +8889,16 @@ class DiffusionBackend:
self._reset_step_cache(state.pipe)
protect_ctx = protect_generation(pipe, denoise_steps, logger = logger)
try:
- # inference_mode is faster than no_grad and numerically identical here.
- with torch.inference_mode(), protect_ctx:
+ # torchao aten.to fails torch's aliasing check under inference_mode; model offload moves weights.
+ with (
+ (
+ torch.no_grad()
+ if state.transformer_quant
+ and state.offload_policy == OFFLOAD_MODEL
+ else torch.inference_mode()
+ ),
+ protect_ctx,
+ ):
out = render_thread.run(
"diffusion", lambda: pipe(**chunk_kwargs).images
)
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index b14cb9d56f8..df347409d49 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -1257,6 +1257,31 @@ def _streamable_components(pipe: Any, torch: Any) -> dict[str, tuple[Any, str]]:
return streamed
+def _module_storage_bytes(module: Any, seen: set[int]) -> int:
+ storage_bytes = 0
+ for tensor in list(module.parameters(recurse = True)) + list(module.buffers(recurse = True)):
+ if id(tensor) in seen:
+ continue
+ seen.add(id(tensor))
+ storage_bytes += int(tensor.numel()) * int(tensor.element_size())
+ return storage_bytes
+
+
+def largest_streamable_companion_mib(pipe: Any) -> Optional[int]:
+ """MiB of the largest text encoder refinement could stream, as loaded, or None."""
+ try:
+ import torch
+ mib = 1024 * 1024
+ sizes = [
+ (_module_storage_bytes(module, set()) + mib - 1) // mib
+ for name, (module, offload_type) in _streamable_components(pipe, torch).items()
+ if offload_type == "leaf_level"
+ ]
+ except Exception: # noqa: BLE001 - a sizing aid; the caller treats unknown as unmeasured
+ return None
+ return max(sizes) if sizes else None
+
+
def refine_memory_plan_for_components(pipe: Any, plan: MemoryPlan) -> MemoryPlan:
"""Replace whole-module offload when a loaded component cannot fit on the device.
@@ -1282,6 +1307,9 @@ def refine_memory_plan_for_components(pipe: Any, plan: MemoryPlan) -> MemoryPlan
transformer = getattr(pipe, "transformer", None)
if not isinstance(components, dict) or not isinstance(transformer, torch.nn.Module):
return plan
+ # Streaming cannot move torchao weights; the loader already checked fit (torchao numel reads bf16-sized here).
+ if _pipe_denoisers_hold_torchao(pipe):
+ return plan
streamable = _streamable_components(pipe, torch)
sizes: dict[str, int] = {}
@@ -1289,18 +1317,7 @@ def refine_memory_plan_for_components(pipe: Any, plan: MemoryPlan) -> MemoryPlan
for name, component in components.items():
if not isinstance(component, torch.nn.Module):
continue
- seen: set[int] = set()
- storage_bytes = 0
- tensors = list(component.parameters(recurse = True)) + list(
- component.buffers(recurse = True)
- )
- for tensor in tensors:
- marker = id(tensor)
- if marker in seen:
- continue
- seen.add(marker)
- storage_bytes += int(tensor.numel()) * int(tensor.element_size())
- sizes[str(name)] = (storage_bytes + mib - 1) // mib
+ sizes[str(name)] = (_module_storage_bytes(component, set()) + mib - 1) // mib
except Exception: # noqa: BLE001 - runtime measurement is an optional refinement
return plan
diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.py
index 7dedc5b76eb..1d6510b5284 100644
--- a/studio/backend/tests/test_diffusion_backend.py
+++ b/studio/backend/tests/test_diffusion_backend.py
@@ -854,8 +854,8 @@ def fake_runtime(monkeypatch):
torch.Generator = _FakeGenerator
torch.cuda = types.SimpleNamespace(is_available = lambda: False)
torch.backends = types.SimpleNamespace(mps = None)
- # generate() wraps the pipe call in torch.inference_mode(); a no-op CM here.
torch.inference_mode = lambda: contextlib.nullcontext()
+ torch.no_grad = lambda: contextlib.nullcontext()
diffusers = types.ModuleType("diffusers")
diffusers.GGUFQuantizationConfig = lambda compute_dtype = None: ("quant", compute_dtype)
@@ -12087,23 +12087,166 @@ def test_a_pipeline_pick_refuses_an_explicit_scheme_that_did_not_engage(
assert "transformer_quant='fp8' could not be used" in str(excinfo.value)
-def test_a_pipeline_pick_does_not_quantise_under_offload(fake_runtime, tmp_path, monkeypatch):
- """Offloaded pipelines stay dense because torchao tensors cannot move."""
+def _offload_plan(
+ offload_policy,
+ budget_mib = 1_000_000,
+ runtime_headroom_mib = 0,
+):
+ real_plan = DiffusionBackend._plan_memory
+
+ def _plan(self, *args, **kwargs):
+ plan = real_plan(self, *args, **kwargs)
+ return dataclasses.replace(
+ plan,
+ offload_policy = offload_policy,
+ estimates = {
+ **plan.estimates,
+ "safe_device_budget_mib": budget_mib,
+ "runtime_headroom_mib": runtime_headroom_mib,
+ "base_overhead_mib": 0,
+ },
+ )
+
+ return _plan
+
+
+def test_a_pipeline_pick_quantises_under_whole_module_offload(fake_runtime, tmp_path, monkeypatch):
backend = DiffusionBackend()
calls = _stub_pipeline_dense_quant(backend, monkeypatch)
+ monkeypatch.setattr(DiffusionBackend, "_plan_memory", _offload_plan("model"))
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512", model_kind = "pipeline", _base_local_dir = str(tmp_path)
+ )
+ assert calls != []
+ assert status["transformer_quant"] is not None
+ backend.unload()
+
+
+ ("offload_policy", "expected"), [("none", "inference_mode"), ("model", "no_grad")]
+)
+def test_an_offloaded_quantised_transformer_renders_outside_inference_mode(
+ fake_runtime, tmp_path, monkeypatch, offload_policy, expected
+):
+ """torchao tensors cannot change device under inference_mode, so offloaded quant renders use no_grad."""
+ import torch
+
+ backend = DiffusionBackend()
+ _stub_pipeline_dense_quant(backend, monkeypatch)
+ monkeypatch.setattr(DiffusionBackend, "_plan_memory", _offload_plan(offload_policy))
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512", model_kind = "pipeline", _base_local_dir = str(tmp_path)
+ )
+ assert status["transformer_quant"] is not None
+ used = []
+
+ def _mode(name):
+ @contextlib.contextmanager
+ def _cm():
+ used.append(name)
+ yield
+
+ return _cm
+
+ monkeypatch.setattr(torch, "inference_mode", _mode("inference_mode"))
+ monkeypatch.setattr(torch, "no_grad", _mode("no_grad"))
+ backend.generate(prompt = "p", steps = 2)
+ assert used == [expected]
+ backend.unload()
+
+
+def test_the_offload_replan_prices_a_precast_text_encoder_once(fake_runtime, tmp_path, monkeypatch):
+ from core.inference import diffusion_te_prequant as te_prequant
+
real_plan = DiffusionBackend._plan_memory
- def _offloading_plan(self, *args, **kwargs):
- plan = real_plan(self, *args, **kwargs)
- return dataclasses.replace(plan, offload_policy = "model")
+ class _Module:
+ def __init__(self, mib):
+ self._t = types.SimpleNamespace(numel = lambda: mib * 1024 * 1024, element_size = lambda: 1)
+
+ def parameters(self, recurse = True):
+ return [self._t]
+
+ def buffers(self, recurse = True):
+ return []
+
+ monkeypatch.setattr(
+ sys.modules["torch"], "nn", types.SimpleNamespace(Module = _Module), raising = False
+ )
+
+ def _replan_te(scale, held_mib = None):
+ monkeypatch.setattr(te_prequant, "te_prequant_budget_scale", lambda *a, **k: scale)
+ backend = DiffusionBackend()
+ _stub_pipeline_dense_quant(backend, monkeypatch)
+ if held_mib is not None:
+ # the pipe already holds the pre-cast encoder: the plan must not price it below the scaled table again
+ monkeypatch.setattr(
+ _FakePipe, "components", {"text_encoder": _Module(held_mib)}, raising = False
+ )
+ seen = []
+
+ def _plan(self, *args, **kwargs):
+ plan = real_plan(self, *args, **kwargs)
+ if kwargs.get("transformer_resident_override_mib") is not None:
+ seen.append(kwargs.get("text_encoder_override_mib"))
+ return dataclasses.replace(plan, offload_policy = "model")
+
+ monkeypatch.setattr(DiffusionBackend, "_plan_memory", _plan)
+ backend.load_pipeline(
+ "Qwen/Qwen-Image-2512", model_kind = "pipeline", _base_local_dir = str(tmp_path)
+ )
+ backend.unload()
+ assert seen, "the quant replan never ran"
+ return seen[-1]
- monkeypatch.setattr(DiffusionBackend, "_plan_memory", _offloading_plan)
+ table_te = _replan_te(1.0)
+ precast_te = _replan_te(0.65)
+ assert precast_te == int(table_te * 0.65)
+ assert _replan_te(0.65, held_mib = table_te // 2) == precast_te
+
+
[email protected]("offload_policy", ["group", "sequential"])
+def test_a_pipeline_pick_stays_dense_under_streamed_offload(
+ fake_runtime, tmp_path, monkeypatch, offload_policy
+):
+ backend = DiffusionBackend()
+ calls = _stub_pipeline_dense_quant(backend, monkeypatch)
+ monkeypatch.setattr(DiffusionBackend, "_plan_memory", _offload_plan(offload_policy))
status = backend.load_pipeline(
"Qwen/Qwen-Image-2512", model_kind = "pipeline", _base_local_dir = str(tmp_path)
)
assert calls == []
assert status["transformer_quant"] is None
- assert "offload" in status["resolved"]["transformer_quant"]["reason"]
+ assert (
+ "hooks torchao weights do not survive" in status["resolved"]["transformer_quant"]["reason"]
+ )
+ backend.unload()
+
+
+ ("budget_mib", "runtime_headroom_mib", "companion_mib"),
+ [(1, 0, None), (1_000_000, 1_000_000, None), (1_000_000, 0, 2_000_000)],
+)
+def test_a_pipeline_pick_stays_dense_when_the_quantised_transformer_exceeds_the_budget(
+ fake_runtime, tmp_path, monkeypatch, budget_mib, runtime_headroom_mib, companion_mib
+):
+ """Quantised transformer (plus runtime headroom) and every text encoder must fit unstreamed."""
+ from core.inference import diffusion as dmod
+
+ backend = DiffusionBackend()
+ calls = _stub_pipeline_dense_quant(backend, monkeypatch)
+ monkeypatch.setattr(dmod, "largest_streamable_companion_mib", lambda pipe: companion_mib)
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_plan_memory",
+ _offload_plan("model", budget_mib = budget_mib, runtime_headroom_mib = runtime_headroom_mib),
+ )
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512", model_kind = "pipeline", _base_local_dir = str(tmp_path)
+ )
+ assert calls == []
+ assert status["transformer_quant"] is None
+ assert "not known to fit" in status["resolved"]["transformer_quant"]["reason"]
backend.unload()
@@ -13235,12 +13378,15 @@ def test_an_explicit_fp8_under_offload_on_nvidia_is_still_declined(
_base_local_dir = str(tmp_path),
)
if fallback == "0":
- with pytest.raises(RuntimeError, match = "Module.to"):
+ with pytest.raises(RuntimeError, match = "hooks torchao weights do not survive"):
backend.load_pipeline("Qwen/Qwen-Image-2512", **kwargs)
else:
status = backend.load_pipeline("Qwen/Qwen-Image-2512", **kwargs)
assert status["transformer_quant"] is None
- assert "Module.to()" in status["resolved"]["transformer_quant"]["reason"]
+ assert (
+ "hooks torchao weights do not survive"
+ in status["resolved"]["transformer_quant"]["reason"]
+ )
backend.unload()
assert calls == [] and reasons == []
@@ -13611,7 +13757,7 @@ def test_a_pipeline_whose_quantised_plan_still_streams_the_transformer_stays_den
_base_local_dir = str(tmp_path),
)
assert calls == []
- assert "Module.to()" in str(excinfo.value)
+ assert "hooks torchao weights do not survive" in str(excinfo.value)
def _gguf_candidate_backend(monkeypatch, tmp_path, *, initial_policy, candidate_plan):
diff --git a/studio/backend/tests/test_diffusion_memory.py b/studio/backend/tests/test_diffusion_memory.py
index 11ed0915b52..269af3d55c6 100644
--- a/studio/backend/tests/test_diffusion_memory.py
+++ b/studio/backend/tests/test_diffusion_memory.py
@@ -793,6 +793,48 @@ def test_refine_model_offload_streams_only_when_a_component_exceeds_budget(monke
assert any("text_encoder" in reason for reason in refined.reasons)
+def test_largest_streamable_companion_mib_measures_only_the_text_encoders(monkeypatch):
+ from core.inference.diffusion_memory import largest_streamable_companion_mib
+
+ Module = _install_sized_torch(monkeypatch)
+ transformer = Module(9000)
+ pipe = types.SimpleNamespace(
+ transformer = transformer,
+ components = {
+ "transformer": transformer,
+ "text_encoder": Module(1200),
+ "text_encoder_2": Module(4800),
+ "vae": Module(9500),
+ },
+ )
+ assert largest_streamable_companion_mib(pipe) == 4800
+ only_dit = types.SimpleNamespace(
+ transformer = transformer, components = {"transformer": transformer}
+ )
+ assert largest_streamable_companion_mib(only_dit) is None
+
+
+def test_refine_keeps_model_offload_for_a_torchao_transformer(monkeypatch):
+ """A torchao transformer's bf16-shaped size must not force streaming."""
+ Module = _install_sized_torch(monkeypatch)
+ plan = MemoryPlan(
+ requested_mode = "low_vram",
+ offload_policy = OFFLOAD_MODEL,
+ vae_tiling = True,
+ vae_slicing = True,
+ device_memory = _discrete(8000),
+ estimates = {"safe_device_budget_mib": 6000},
+ )
+ transformer = Module(7500)
+ pipe = types.SimpleNamespace(
+ transformer = transformer,
+ components = {"transformer": transformer, "text_encoder": Module(1200)},
+ )
+ assert refine_memory_plan_for_components(pipe, plan).offload_policy == OFFLOAD_STREAMING
+ monkeypatch.setattr(diffusion_memory, "_pipe_denoisers_hold_torchao", lambda pipe: True)
+ assert refine_memory_plan_for_components(pipe, plan) is plan
+
+
def test_refine_keeps_model_offload_when_streaming_cannot_help(monkeypatch):
"""Only a component streaming can actually hook justifies leaving whole-module offload."""
Module = _install_sized_torch(monkeypatch)
diff --git a/studio/backend/tests/test_diffusion_native_quant.py b/studio/backend/tests/test_diffusion_native_quant.py
index 28c52d83595..013f6de9010 100644
--- a/studio/backend/tests/test_diffusion_native_quant.py
+++ b/studio/backend/tests/test_diffusion_native_quant.py
@@ -828,14 +828,23 @@ def test_image_gate_admits_int8_under_offload_on_nvidia(nvidia, monkeypatch, mem
@pytest.mark.parametrize("pinned", ["fp8", "nvfp4", "mxfp8"])
[email protected]("memory", [{"memory_mode": "balanced"}, {"cpu_offload": True}])
-def test_image_gate_still_refuses_other_schemes_under_offload_on_nvidia(
+def test_image_gate_still_refuses_other_schemes_under_balanced_on_nvidia(
+ nvidia, monkeypatch, pinned
+):
+ import core.inference.diffusion as d
+ _torchao_gate_answers(monkeypatch, d)
+ with pytest.raises(RuntimeError, match = "hooks torchao weights do not survive"):
+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), pinned, memory_mode = "balanced")
+
+
[email protected]("pinned", ["fp8", "nvfp4", "mxfp8"])
[email protected]("memory", [{"memory_mode": "low_vram"}, {"cpu_offload": True}])
+def test_image_gate_admits_other_schemes_under_whole_module_offload_on_nvidia(
nvidia, monkeypatch, pinned, memory
):
import core.inference.diffusion as d
_torchao_gate_answers(monkeypatch, d)
- with pytest.raises(RuntimeError, match = "cannot be moved by the offload hooks"):
- _image_gate(monkeypatch, _image_family("qwen-image-2.1"), pinned, **memory)
+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), pinned, **memory)
@pytest.mark.parametrize("kind", ["gguf", "single_file"])
diff --git a/studio/backend/tests/test_diffusion_nvfp4_protect_image.py b/studio/backend/tests/test_diffusion_nvfp4_protect_image.py
index 0fb645e61cf..7b1b0b91727 100644
--- a/studio/backend/tests/test_diffusion_nvfp4_protect_image.py
+++ b/studio/backend/tests/test_diffusion_nvfp4_protect_image.py
@@ -71,12 +71,14 @@ def test_the_lever_wraps_the_chunk_render_itself():
continue
names = set()
for item in node.items:
- expr = item.context_expr
- if isinstance(expr, ast.Name):
- names.add(expr.id)
- elif isinstance(expr, ast.Call):
- func = expr.func
- names.add(func.attr if isinstance(func, ast.Attribute) else getattr(func, "id", ""))
+ for expr in ast.walk(item.context_expr):
+ if isinstance(expr, ast.Name):
+ names.add(expr.id)
+ elif isinstance(expr, ast.Call):
+ func = expr.func
+ names.add(
+ func.attr if isinstance(func, ast.Attribute) else getattr(func, "id", "")
+ )
if "protect_ctx" in names:
wrapped.append(names)
assert wrapped, "the protect context is never entered"
diff --git a/studio/backend/tests/test_diffusion_routes.py b/studio/backend/tests/test_diffusion_routes.py
index 46174572f08..8fba0a10082 100644
--- a/studio/backend/tests/test_diffusion_routes.py
+++ b/studio/backend/tests/test_diffusion_routes.py
@@ -2201,10 +2201,7 @@ def test_cancel_generation_route_requires_auth():
)
def test_an_offloading_memory_request_refuses_an_explicit_precision(monkeypatch, memory):
"""balanced and low_vram name their offload policy outright, and the legacy cpu_offload flag
- forces whole-module offload. Offload hooks move modules with Module.to(), which torchao
- tensors do not survive, so the loader skips the dense build -- and the strict refusal then
- arrived after the resident image model had already been torn down. The two requests are
- incompatible on their face, so the refusal is owed before anything is staged or evicted."""
+ forces whole-module offload, so the refusal is owed before anything is staged or evicted."""
from core.inference.diffusion import DiffusionBackend
backend = DiffusionBackend.__new__(DiffusionBackend)
@@ -2221,6 +2218,38 @@ def test_an_offloading_memory_request_refuses_an_explicit_precision(monkeypatch,
assert "offload" in str(excinfo.value)
+def _pipeline_precision_gate(monkeypatch, **memory):
+ from core.inference.diffusion import DiffusionBackend
+
+ backend = DiffusionBackend.__new__(DiffusionBackend)
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_resolve_device_target",
+ lambda self, fam: _cuda_target(),
+ )
+ monkeypatch.setattr(diffusion_module, "dense_transformer_supported", lambda target: True)
+ monkeypatch.setattr(diffusion_module, "select_transformer_quant_scheme", lambda *a, **k: "fp8")
+ backend.assert_precision_available(
+ types.SimpleNamespace(name = "z-image"),
+ model_kind = "pipeline",
+ transformer_quant = "fp8",
+ **memory,
+ )
+
+
[email protected]("memory", [{"memory_mode": "low_vram"}, {"cpu_offload": True}])
+def test_a_whole_module_offload_request_admits_a_pipeline_precision(monkeypatch, memory):
+ """low_vram and cpu_offload pick whole-module offload, which a pipeline quantises under."""
+ _pipeline_precision_gate(monkeypatch, **memory)
+
+
+def test_balanced_memory_refuses_a_pipeline_precision(monkeypatch):
+ """balanced is group offload, which the loader never quantises under."""
+ with pytest.raises(RuntimeError) as excinfo:
+ _pipeline_precision_gate(monkeypatch, memory_mode = "balanced")
+ assert "group-offload hooks" in str(excinfo.value)
+
+
def test_a_measured_memory_mode_is_not_refused_by_the_precision_gate(monkeypatch):
"""fast and auto decide their policy from the MEASURED footprint, so they are not knowable
network-free and this gate has no business refusing them."""