Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep bf16 for auto precision on image families that cannot compile (#11819)

Daniel Hane5082afd43e2
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index f3732435e07..5c35347921d 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -131,6 +131,7 @@ from .diffusion_speed import (
compile_dynamic,
compile_eligible,
compiled_shapes_are_static,
+ family_compiles_regionally,
fp16_compile_explicit_only,
fp16_unet_offloaded,
fresh_compile_count,
@@ -192,6 +193,7 @@ from .diffusion_denoiser_prequant import (
from .diffusion_prequant import (
hosted_fast_accum_conflict,
load_prequantized_transformer,
+ local_prequant_path_ready,
prequant_checkpoint_cached,
prequant_unreadable_reason,
resolve_prequant_source,
@@ -1422,6 +1424,31 @@ def _local_base_transformer_present(base_repo: Optional[str]) -> bool:
return False
+_UNSHARDED_PIPELINE_WEIGHT_RE = re.compile(
+ r"^(?:diffusion_pytorch_model|pytorch_model|model)\.(?:bin|safetensors)$"
+)
+
+
+def _transformer_folder_complete(folder: Path) -> bool:
+ """Complete checkpoint for diffusers' default load: safetensors index first, never a ``.bin`` index."""
+ if not folder.is_dir():
+ return False
+ names = {f.name for f in folder.iterdir() if f.is_file()}
+ indexes = [
+ n
+ for n in names
+ if n.endswith(".safetensors.index.json") and _DEFAULT_PIPELINE_WEIGHT_INDEX_RE.match(n)
+ ]
+ if indexes:
+ for index in indexes:
+ with open(folder / index, encoding = "utf-8") as fh:
+ shards = set((json.load(fh).get("weight_map") or {}).values())
+ if shards and shards <= names:
+ return True
+ return False
+ return any(_UNSHARDED_PIPELINE_WEIGHT_RE.match(n) for n in names)
+
+
def _dense_candidate_is_prequant(
fam: Optional[DiffusionFamily],
target: Any,
@@ -1527,6 +1554,37 @@ def _pipeline_quant_uncompilable_reason(
return None
+def _plan_proves_resident(plan: Any) -> bool:
+ """Resident AND fits; the planner also stays resident when it cannot read the card."""
+ estimates = getattr(plan, "estimates", None) or {}
+ budget = estimates.get("safe_device_budget_mib")
+ required = estimates.get("resident_required_mib")
+ # Unified memory plans 'none' whatever the size (offload frees nothing there), so compare the two as well.
+ return (
+ getattr(plan, "offload_policy", None) == OFFLOAD_NONE
+ and budget is not None
+ and required is not None
+ and int(required) <= int(budget)
+ )
+
+
+def _auto_quant_eager_reason(
+ fam: Any,
+ plan: Any,
+ prequant_path: Optional[str] = None,
+) -> Optional[str]:
+ """Why AUTO keeps bf16 for an uncompilable family, or None; an operator's own checkpoint wins."""
+ if not _plan_proves_resident(plan) or family_compiles_regionally(fam):
+ return None
+ if prequant_path and local_prequant_path_ready(prequant_path):
+ return None
+ return (
+ f"'{getattr(fam, 'name', None)}' cannot be regionally compiled (its transformer declares no "
+ "repeated blocks), and an uncompiled quantised transformer runs far slower than the weights "
+ "it replaces; they fit on this GPU as they are"
+ )
+
+
def _clear_exception_frames(exc: BaseException) -> None:
"""Release failed-call locals while preserving traceback locations."""
errors, seen = [exc], set()
@@ -2647,6 +2705,8 @@ class DiffusionBackend:
gpu_ordinal = kwargs.get("gpu_ordinal"),
repo_id = kwargs["repo_id"],
fast_accum = kwargs.get("transformer_quant_fast_accum"),
+ text_encoder_quant = te_quant_planned if te_prequant_files else None,
+ local_files_only = local_files_only,
)
if local_files_only and pipeline_planned not in (None, PIPELINE_SEED_DECLINED):
# Offline twin of the Hub probe below. The first load left the released shards out of the cache,
@@ -2725,6 +2785,7 @@ class DiffusionBackend:
)
)
kwargs["_pipeline_prequant_skipped"] = tuple(skipped_transformer_files)
+ kwargs["_te_prequant_resolved"] = bool(te_prequant_files)
if dit_prequant is not None:
expected += int(dit_prequant[2])
# Only shards this prefetch staged may be materialised by the dense fallback, so read it off the staged
@@ -3019,6 +3080,8 @@ class DiffusionBackend:
repo_id: Optional[str] = None,
fetch_base: Optional[str] = None,
fast_accum: Optional[bool] = None,
+ text_encoder_quant: Optional[str] = None,
+ local_files_only: bool = False,
) -> Optional[str]:
"""The scheme an official ``kind == "pipeline"`` pick seeds its denoiser from, settled BEFORE
anything is downloaded; None keeps the released bf16 weights and ``PIPELINE_SEED_DECLINED``
@@ -3052,6 +3115,42 @@ class DiffusionBackend:
is not None
):
return None
+ if auto and not family_compiles_regionally(fam):
+ bf16_memory = snapshot_device_memory(target)
+ bf16_plan = self._bf16_table_plan(
+ target,
+ fam,
+ base or repo_id or "",
+ memory_mode,
+ cpu_offload,
+ kind = kind,
+ repo_id = repo_id,
+ fetch_base = fetch_base,
+ text_encoder_quant = None if local_files_only else text_encoder_quant,
+ device_memory_override = replace(bf16_memory, free_mib = bf16_memory.total_mib),
+ )
+ # Offline without the released shards, bf16 cannot assemble: a cached seed is the only way.
+ if (
+ bf16_plan is not None
+ and _auto_quant_eager_reason(fam, bf16_plan, transformer_prequant_path)
+ is not None
+ and not (
+ local_files_only
+ and not any(
+ self._released_transformer_cached(candidate)
+ for candidate in dict.fromkeys(
+ (
+ fetch_base,
+ base,
+ repo_id,
+ prefer_ungated_mirror(base or repo_id or ""),
+ )
+ )
+ if candidate
+ )
+ )
+ ):
+ return None
scheme = select_transformer_quant_scheme(
target, mode, family = getattr(fam, "name", None)
)
@@ -3570,6 +3669,7 @@ class DiffusionBackend:
gpu_ordinal = load_kwargs.get("gpu_ordinal"),
repo_id = repo_id,
fast_accum = load_kwargs.get("transformer_quant_fast_accum"),
+ text_encoder_quant = te_quant_planned if te_files else None,
)
if allow_device_probe
else None
@@ -4406,6 +4506,7 @@ class DiffusionBackend:
# The scheme the plan settled, or PIPELINE_SEED_DECLINED; None for a direct call, which the pull never scoped.
_pipeline_prequant_planned: Optional[str] = None,
_pipeline_prequant_skipped: tuple[str, ...] = (),
+ _te_prequant_resolved: bool = True,
) -> dict[str, Any]:
with self._load_cancel_lock:
if _load_token is None:
@@ -4624,6 +4725,44 @@ class DiffusionBackend:
# visible, and into the refusal so it is actionable.
transformer_quant_decline: Optional[str] = None
transformer_quant_decline_status = RESOLVED_FELL_BACK
+ # A GGUF is excluded while it bakes LoRAs: only the dense build can carry them.
+ if (
+ transformer_quant == TQ_AUTO
+ and pipeline_seed_scheme is None
+ and dense_quant_supported_kind(kind)
+ and dense_transformer_supported(target)
+ and not (kind == "gguf" and _has_active_lora(loras))
+ and not family_compiles_regionally(fam)
+ and (
+ eager_reason := _auto_quant_eager_reason(
+ fam,
+ eager_plan := self._bf16_resident_plan(
+ plan,
+ target,
+ fam,
+ base,
+ memory_mode,
+ cpu_offload,
+ kind = kind,
+ repo_id = repo_id,
+ fetch_base = fetch_base,
+ base_local_dir = _base_local_dir,
+ text_encoder_quant = text_encoder_quant
+ if _te_prequant_resolved and not local_files_only
+ else None,
+ ),
+ transformer_prequant_path,
+ )
+ )
+ is not None
+ ):
+ logger.info(
+ "diffusion.transformer_quant: auto keeps %s (%s)", kind, eager_reason
+ )
+ transformer_quant = "off"
+ transformer_quant_decline = eager_reason
+ # Place the released weights by the plan that proved they fit, not by fp32 shard bytes.
+ plan = eager_plan
native_scheme = (
native_quant_scheme(
target, transformer_quant_pinned, family = getattr(fam, "name", None)
@@ -6459,6 +6598,137 @@ class DiffusionBackend:
pipe.to(device)
return pipe
+ def _bf16_table_plan(
+ self,
+ target: DiffusionDeviceTarget,
+ fam: DiffusionFamily,
+ base: str,
+ memory_mode: Optional[str],
+ cpu_offload: bool,
+ *,
+ kind: str,
+ repo_id: Optional[str],
+ fetch_base: Optional[str] = None,
+ base_local_dir: Optional[str] = None,
+ text_encoder_quant: Optional[str] = None,
+ device_memory_override: Optional[DeviceMemory] = None,
+ ) -> Any:
+ """Plan priced from the family's bf16-resident table, or None for an unknown family."""
+ table = family_bf16_components_gb(fam, base)
+ if table is None:
+ return None
+ text_encoder_gb = table[1]
+ try:
+ from .diffusion_te_prequant import (
+ TE_PREQUANT_BUDGET_SCALE,
+ te_prequant_budget_scale,
+ te_prequant_sources_for_base,
+ )
+
+ te_scale = te_prequant_budget_scale(
+ fam, te_quant_mode = text_encoder_quant, target = target, base = base
+ )
+ text_encoder_gb = table[1] * te_scale
+ # HiDream's Llama text_encoder_4 comes pre-cast from its own repo, outside the components the scale probes.
+ if (
+ te_scale == 1.0
+ and getattr(fam, "name", None) == HIDREAM_FAMILY_NAME
+ and te_prequant_sources_for_base(
+ fam,
+ HIDREAM_LLAMA_REPO,
+ te_quant_mode = text_encoder_quant,
+ target = target,
+ components = ("text_encoder_4",),
+ standalone_component_bases = {"text_encoder_4": HIDREAM_LLAMA_REPO},
+ ).get("text_encoder_4")
+ is not None
+ ):
+ llama_gb = min(HIDREAM_LLAMA_BF16_BYTES / 1000.0**3, table[1])
+ text_encoder_gb = table[1] - llama_gb * (1.0 - TE_PREQUANT_BUDGET_SCALE)
+ except Exception: # noqa: BLE001 -- sizing aid only; budget the dense encoder
+ text_encoder_gb = table[1]
+ mib_per_gb = 1000.0**3 / (1024.0 * 1024.0)
+ text_encoder_mib = int(text_encoder_gb * mib_per_gb)
+ return self._plan_memory(
+ target,
+ None,
+ base,
+ fam,
+ memory_mode,
+ cpu_offload,
+ kind = kind,
+ repo_id = repo_id,
+ fetch_base = fetch_base,
+ base_local_dir = base_local_dir,
+ transformer_resident_override_mib = int(table[0] * mib_per_gb),
+ companion_override_mib = text_encoder_mib + int(table[2] * mib_per_gb),
+ text_encoder_override_mib = text_encoder_mib,
+ device_memory_override = device_memory_override,
+ )
+
+ def _bf16_resident_plan(
+ self,
+ plan: Any,
+ target: DiffusionDeviceTarget,
+ fam: DiffusionFamily,
+ base: str,
+ memory_mode: Optional[str],
+ cpu_offload: bool,
+ *,
+ kind: str,
+ repo_id: Optional[str],
+ fetch_base: Optional[str] = None,
+ base_local_dir: Optional[str] = None,
+ text_encoder_quant: Optional[str] = None,
+ ) -> Any:
+ """``plan`` re-priced when the table lowers it (cached shards wider than bf16).
+
+ HiDream always: no cached plan counts its separately loaded text_encoder_4."""
+ if kind != "pipeline":
+ return plan
+ if (
+ getattr(fam, "name", None) != HIDREAM_FAMILY_NAME
+ and self._resident_sized_plan(
+ plan, fam, base, target, kind, text_encoder_quant = text_encoder_quant
+ )
+ is plan
+ ):
+ return plan
+ try:
+ repriced = self._bf16_table_plan(
+ target,
+ fam,
+ base,
+ memory_mode,
+ cpu_offload,
+ kind = kind,
+ repo_id = repo_id,
+ fetch_base = fetch_base,
+ base_local_dir = base_local_dir,
+ text_encoder_quant = text_encoder_quant,
+ device_memory_override = getattr(plan, "device_memory", None),
+ )
+ except Exception: # noqa: BLE001 -- sizing aid only; decide on the plan as built
+ return plan
+ return plan if repriced is None else repriced
+
+ @staticmethod
+ def _released_transformer_cached(base: Optional[str]) -> bool:
+ """Complete ``transformer/`` in the snapshot the loader reads; roots in ``_assert_base_repo_accessible`` order."""
+ if not base:
+ return False
+ try:
+ local = Path(base).expanduser()
+ if local.is_dir():
+ return _transformer_folder_complete(local / "transformer")
+ for repo_dir in DiffusionBackend._hub_cache_repo_dirs(base):
+ snapshot = DiffusionBackend._live_snapshot_dir(repo_dir)
+ if snapshot is not None and (snapshot / "model_index.json").is_file():
+ return _transformer_folder_complete(snapshot / "transformer")
+ return False
+ except Exception: # noqa: BLE001 -- unreadable cache: treat the shards as missing
+ return False
+
@staticmethod
def _precast_scaled_companions_mib(
candidate: Any,
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.py
index 3af547b117a..bb64b3fea3c 100644
--- a/studio/backend/core/inference/diffusion_speed.py
+++ b/studio/backend/core/inference/diffusion_speed.py
@@ -283,6 +283,30 @@ def fp16_compile_explicit_only(target: Any) -> bool:
return _is_float16(getattr(target, "dtype", None))
+def family_compiles_regionally(family: Any) -> bool:
+ """False only for an EMPTY ``_repeated_blocks`` (``compile_repeated_blocks`` raises); unknown reads True."""
+ if getattr(family, "denoiser_attr", "transformer") != "transformer":
+ return True
+ name = getattr(family, "transformer_class", None)
+ if not isinstance(name, str) or not name:
+ return True
+ # Called before load_pipeline's guard; `import diffusers` imports torch._dynamo, so wait for the torch warm.
+ try:
+ from loggers import get_logger
+ from utils.torch_warmup import close_dynamo_import_window
+ close_dynamo_import_window(get_logger(__name__))
+ except Exception: # noqa: BLE001, S110 - optimisation only
+ pass
+ try:
+ import diffusers
+ cls = getattr(diffusers, name, None)
+ except Exception: # noqa: BLE001 - an unanswerable probe keeps today's behaviour
+ return True
+ if cls is None or not hasattr(cls, "_repeated_blocks"):
+ return True
+ return bool(cls._repeated_blocks)
+
+
def _is_bfloat16(dtype: Any) -> bool:
try:
import torch
diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.py
index a5ad1fee84c..4f62104cb77 100644
--- a/studio/backend/tests/test_diffusion_backend.py
+++ b/studio/backend/tests/test_diffusion_backend.py
@@ -5248,6 +5248,19 @@ def test_declined_dense_without_loras_still_falls_back_to_gguf(
assert backend.status()["transformer_quant"] is None # GGUF-as-is fallback
+def test_an_uncompilable_gguf_baking_loras_keeps_the_dense_build(
+ fake_runtime, tmp_path, monkeypatch
+):
+ from core.inference import diffusion as dmod
+
+ backend = DiffusionBackend()
+ _decline_dense_quant(backend, monkeypatch, tmp_path)
+ monkeypatch.setattr(dmod, "family_compiles_regionally", lambda _fam: False)
+ monkeypatch.setattr(dmod, "_plan_proves_resident", lambda _plan: True)
+ with pytest.raises(RuntimeError, match = "LoRA adapters could not be applied"):
+ _load_m(backend, tmp_path, loras = [("adapter", 1.0)])
+
+
class _BakePipe:
def __init__(self):
self.calls: list = []
diff --git a/studio/backend/tests/test_diffusion_pipeline_prequant.py b/studio/backend/tests/test_diffusion_pipeline_prequant.py
index a6f4f9bb458..6206802e794 100644
--- a/studio/backend/tests/test_diffusion_pipeline_prequant.py
+++ b/studio/backend/tests/test_diffusion_pipeline_prequant.py
@@ -386,9 +386,9 @@ def _settle_backend_walking(monkeypatch, *, artifacts: tuple, candidates: tuple)
monkeypatch.setattr(
dmod,
"denoiser_prequant_source",
- lambda fam, scheme, **_k: ("unsloth/Qwen-Image-FP8", f"{scheme}.pt")
- if scheme in artifacts
- else None,
+ lambda fam, scheme, **_k: (
+ ("unsloth/Qwen-Image-FP8", f"{scheme}.pt") if scheme in artifacts else None
+ ),
)
monkeypatch.setattr(
dmod,
@@ -524,9 +524,11 @@ def _run_load_backend(
monkeypatch.setattr(
DiffusionBackend,
"_dit_prequant_plan_source",
- lambda *_a, **_k: (PREQUANT_REPO, PREQUANT_FILE, PREQUANT_BYTES)
- if verified and planned not in (None, PIPELINE_SEED_DECLINED)
- else None,
+ lambda *_a, **_k: (
+ (PREQUANT_REPO, PREQUANT_FILE, PREQUANT_BYTES)
+ if verified and planned not in (None, PIPELINE_SEED_DECLINED)
+ else None
+ ),
)
fetched: list = []
monkeypatch.setattr(
@@ -953,3 +955,601 @@ def test_the_rebuilt_fp8_artifacts_are_listed_for_both_schemes(family, repo):
assert fam is not None
for scheme in ("fp8", "int8"):
assert family_prequant_repo(fam, scheme) == repo
+
+
+_MEASURED = {"safe_device_budget_mib": 170_000, "resident_required_mib": 60_000}
+
+
+def _uncompilable(monkeypatch, *, measured = True):
+ monkeypatch.setattr(dmod, "family_compiles_regionally", lambda _fam: False)
+ monkeypatch.setattr(dmod, "family_bf16_components_gb", lambda *_a, **_k: (90.0, 10.0, 0.2))
+ inner = DiffusionBackend._plan_memory
+
+ def _plan(self, *a, **k):
+ plan = inner(self, *a, **k)
+ plan.estimates = dict(_MEASURED) if measured else {}
+ return plan
+
+ monkeypatch.setattr(DiffusionBackend, "_plan_memory", _plan)
+
+
+def _offload_when_bf16_sized(monkeypatch, bf16_policy):
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_plan_memory",
+ lambda *_a, **k: types.SimpleNamespace(
+ offload_policy = bf16_policy
+ if (k.get("transformer_resident_override_mib") or 0) > 80_000
+ else "none"
+ ),
+ )
+
+
+def test_an_uncompilable_family_keeps_the_released_weights_that_fit(monkeypatch):
+ backend = _settle_backend(monkeypatch)
+ _offload_when_bf16_sized(monkeypatch, "none")
+ _uncompilable(monkeypatch)
+ assert _settle(backend) is None
+
+
+def test_an_uncompilable_family_still_seeds_when_the_budget_is_unmeasured(monkeypatch):
+ backend = _settle_backend(monkeypatch)
+ _offload_when_bf16_sized(monkeypatch, "none")
+ _uncompilable(monkeypatch, measured = False)
+ assert _settle(backend) == "fp8"
+
+
+def test_an_uncompilable_family_still_seeds_when_the_released_weights_would_offload(monkeypatch):
+ backend = _settle_backend(monkeypatch)
+ _offload_when_bf16_sized(monkeypatch, "sequential")
+ _uncompilable(monkeypatch)
+ assert _settle(backend) == "fp8"
+
+
+def test_an_uncompilable_family_still_seeds_an_explicit_scheme(monkeypatch):
+ backend = _settle_backend(monkeypatch)
+ _offload_when_bf16_sized(monkeypatch, "none")
+ _uncompilable(monkeypatch)
+ assert _settle(backend, transformer_quant = "fp8") == "fp8"
+
+
+def _operator_checkpoint(monkeypatch, tmp_path):
+ ckpt = tmp_path / "transformer_fp8.pt"
+ ckpt.write_bytes(b"x")
+ monkeypatch.setenv(pqmod.ALLOW_LOCAL_PREQUANT_PATH_ENV, str(tmp_path))
+ return str(ckpt)
+
+
+def test_an_uncompilable_family_still_plans_an_operators_checkpoint(monkeypatch, tmp_path):
+ backend = _settle_backend(monkeypatch)
+ _offload_when_bf16_sized(monkeypatch, "none")
+ _uncompilable(monkeypatch)
+ path = _operator_checkpoint(monkeypatch, tmp_path)
+ monkeypatch.setattr(pqmod, "local_prequant_scheme", lambda _path: "fp8")
+ assert _settle(backend, transformer_prequant_path = path) is not None
+ # An unloadable path is no ask: the released weights still win.
+ assert _settle(backend, transformer_prequant_path = str(tmp_path / "missing.pt")) is None
+
+
+def test_a_compilable_family_seeds_as_before(monkeypatch):
+ backend = _settle_backend(monkeypatch)
+ monkeypatch.setattr(dmod, "family_compiles_regionally", lambda _fam: True)
+ _offload_when_bf16_sized(monkeypatch, "none")
+ assert _settle(backend) == "fp8"
+
+
+def test_an_uncompilable_family_loads_auto_unquantised(fake_runtime, monkeypatch):
+ backend, spy = _load_backend(monkeypatch)
+ _uncompilable(monkeypatch)
+ status = _load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())
+
+ assert spy.quantised == []
+ assert spy.seeds == []
+ resolved = status["resolved"]["transformer_quant"]
+ assert (resolved["value"], resolved["source"], resolved["status"]) == ("off", "auto", "applied")
+ assert "cannot be regionally compiled" in resolved["reason"]
+
+
+def test_an_uncompilable_family_loads_an_operators_checkpoint_under_auto(
+ fake_runtime, monkeypatch, tmp_path
+):
+ backend, spy = _load_backend(monkeypatch)
+ _uncompilable(monkeypatch)
+ status = _load(
+ backend,
+ transformer_prequant_path = _operator_checkpoint(monkeypatch, tmp_path),
+ _pipeline_prequant_planned = None,
+ _pipeline_prequant_skipped = (),
+ )
+
+ assert spy.quantised == ["auto"]
+ assert "cannot be regionally compiled" not in str(status["resolved"]["transformer_quant"])
+
+
+def test_a_compilable_family_loads_without_the_bf16_replan(fake_runtime, monkeypatch):
+ backend, spy = _load_backend(monkeypatch)
+ monkeypatch.setattr(dmod, "family_compiles_regionally", lambda _fam: True)
+ replans = []
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_bf16_resident_plan",
+ lambda self, plan, *a, **k: replans.append(1) or plan,
+ )
+ _load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())
+
+ assert replans == []
+ assert spy.quantised == ["auto"]
+
+
+def test_an_uncompilable_family_quantises_auto_when_bf16_would_offload(fake_runtime, monkeypatch):
+ backend, spy = _load_backend(monkeypatch)
+ _uncompilable(monkeypatch)
+ real_plan = DiffusionBackend._plan_memory
+
+ def _plan(self, *a, **k):
+ plan = real_plan(self, *a, **k)
+ if k.get("transformer_resident_override_mib") is None:
+ plan.offload_policy = "sequential"
+ return plan
+
+ monkeypatch.setattr(DiffusionBackend, "_plan_memory", _plan)
+ _load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())
+
+ assert spy.quantised == ["auto"]
+
+
+def test_an_uncompilable_family_honours_an_explicit_scheme(fake_runtime, monkeypatch):
+ backend, spy = _load_backend(monkeypatch)
+ _uncompilable(monkeypatch)
+ _load(
+ backend,
+ transformer_quant = "fp8",
+ _pipeline_prequant_planned = None,
+ _pipeline_prequant_skipped = (),
+ )
+
+ assert spy.quantised == ["fp8"]
+
+
+def test_an_uncompilable_family_on_a_host_without_dense_quant_reports_as_before(
+ fake_runtime, monkeypatch
+):
+ backend, spy = _load_backend(monkeypatch)
+ _uncompilable(monkeypatch)
+ monkeypatch.setattr(dmod, "dense_transformer_supported", lambda _t: False)
+ status = _load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())
+
+ assert spy.quantised == []
+ assert "regionally compiled" not in status["resolved"]["transformer_quant"]["reason"]
+
+
+def test_an_uncompilable_family_quantises_auto_when_the_budget_is_unmeasured(
+ fake_runtime, monkeypatch
+):
+ backend, spy = _load_backend(monkeypatch)
+ _uncompilable(monkeypatch, measured = False)
+ _load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())
+
+ assert spy.quantised == ["auto"]
+
+
+def test_a_resident_plan_whose_requirement_exceeds_the_budget_proves_no_fit():
+ fits = types.SimpleNamespace(
+ offload_policy = "none",
+ estimates = {"safe_device_budget_mib": 60_000, "resident_required_mib": 60_000},
+ )
+ over = types.SimpleNamespace(
+ offload_policy = "none",
+ estimates = {"safe_device_budget_mib": 50_000, "resident_required_mib": 60_000},
+ )
+ assert dmod._plan_proves_resident(fits)
+ assert not dmod._plan_proves_resident(over)
+
+
+def test_an_uncompilable_family_seeds_when_bf16_overflows_a_unified_pool(monkeypatch):
+ backend = _settle_backend(monkeypatch)
+ monkeypatch.setattr(dmod, "family_compiles_regionally", lambda _fam: False)
+ monkeypatch.setattr(dmod, "family_bf16_components_gb", lambda *_a, **_k: (90.0, 10.0, 0.2))
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_plan_memory",
+ lambda *_a, **_k: types.SimpleNamespace(
+ offload_policy = "none",
+ estimates = {"safe_device_budget_mib": 50_000, "resident_required_mib": 60_000},
+ ),
+ )
+ assert _settle(backend) == "fp8"
+
+
+def test_an_uncompilable_family_prices_a_pre_cast_text_encoder(monkeypatch):
+ import core.inference.diffusion_te_prequant as teq
+
+ backend = _settle_backend(monkeypatch)
+ monkeypatch.setattr(
+ teq,
+ "te_prequant_budget_scale",
+ lambda _fam, *, te_quant_mode, target, base: 0.5 if te_quant_mode == "fp8" else 1.0,
+ )
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_plan_memory",
+ lambda *_a, **k: types.SimpleNamespace(
+ offload_policy = "none"
+ if (k.get("companion_override_mib") or 0) < 6_000
+ or (k.get("transformer_resident_override_mib") or 0) < 80_000
+ else "sequential"
+ ),
+ )
+ _uncompilable(monkeypatch)
+ assert _settle(backend) == "fp8"
+ assert _settle(backend, text_encoder_quant = "fp8") is None
+
+
+def test_an_offline_uncompilable_pick_keeps_a_cached_seed_without_released_shards(monkeypatch):
+ backend = _settle_backend(monkeypatch)
+ _offload_when_bf16_sized(monkeypatch, "none")
+ _uncompilable(monkeypatch)
+ monkeypatch.setattr(
+ DiffusionBackend, "_released_transformer_cached", staticmethod(lambda _base: False)
+ )
+ assert _settle(backend, local_files_only = True) == "fp8"
+ assert _settle(backend) is None
+ monkeypatch.setattr(
+ DiffusionBackend, "_released_transformer_cached", staticmethod(lambda _base: True)
+ )
+ assert _settle(backend, local_files_only = True) is None
+
+
+def test_released_transformer_cached_reads_the_transformer_folder(tmp_path):
+ (tmp_path / "vae").mkdir()
+ (tmp_path / "vae" / "diffusion_pytorch_model.safetensors").write_bytes(b"x")
+ assert not DiffusionBackend._released_transformer_cached(str(tmp_path))
+ (tmp_path / "transformer").mkdir()
+ (tmp_path / "transformer" / "diffusion_pytorch_model.safetensors").write_bytes(b"x")
+ assert DiffusionBackend._released_transformer_cached(str(tmp_path))
+ assert not DiffusionBackend._released_transformer_cached(None)
+
+
+def test_the_seed_planner_hears_the_encoder_choice_and_offline_flag(monkeypatch):
+ backend, _seen, _fetched = _run_load_backend(monkeypatch, planned = None)
+ heard: dict = {}
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_pipeline_planned_denoiser_scheme",
+ lambda _self, _fam, **k: heard.update(k),
+ )
+ backend._run_load(
+ repo_id = Z_IMAGE_REPO, model_kind = "pipeline", local_files_only = True, _load_token = 1
+ )
+ assert heard["local_files_only"] is True
+ assert "text_encoder_quant" in heard
+
+
+def test_an_uncompilable_family_keeps_bf16_when_the_resident_table_fits(fake_runtime, monkeypatch):
+ backend, spy = _load_backend(monkeypatch)
+ _uncompilable(monkeypatch)
+ real_plan = DiffusionBackend._plan_memory
+ overrides: list = []
+
+ def _plan(self, *a, **k):
+ plan = real_plan(self, *a, **k)
+ overrides.append(k.get("transformer_resident_override_mib"))
+ if k.get("transformer_resident_override_mib") is None:
+ plan.offload_policy = "sequential"
+ return plan
+
+ monkeypatch.setattr(DiffusionBackend, "_plan_memory", _plan)
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_resident_sized_plan",
+ lambda _s, plan, *_a, **_k: types.SimpleNamespace(**vars(plan)),
+ )
+ status = _load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())
+
+ assert spy.quantised == []
+ assert int(90.0 * 1000.0**3 / (1024.0 * 1024.0)) in overrides
+ assert status["resolved"]["transformer_quant"]["value"] == "off"
+
+
+def _write_index(folder, shards):
+ folder.mkdir(parents = True, exist_ok = True)
+ (folder / "diffusion_pytorch_model.safetensors.index.json").write_text(
+ json.dumps({"weight_map": {f"w{i}": shard for i, shard in enumerate(shards)}})
+ )
+
+
+def test_a_partial_sharded_transformer_is_not_a_cached_release(tmp_path):
+ shards = [f"diffusion_pytorch_model-0000{i}-of-00002.safetensors" for i in (1, 2)]
+ _write_index(tmp_path / "transformer", shards)
+ (tmp_path / "transformer" / shards[0]).write_bytes(b"x")
+ assert not DiffusionBackend._released_transformer_cached(str(tmp_path))
+ (tmp_path / "transformer" / shards[1]).write_bytes(b"x")
+ assert DiffusionBackend._released_transformer_cached(str(tmp_path))
+
+
+def test_a_complete_bin_set_does_not_cover_a_partial_safetensors_index(tmp_path):
+ # Diffusers loads the safetensors index when present and never a .bin index by default.
+ folder = tmp_path / "transformer"
+ bins = [f"diffusion_pytorch_model-0000{i}-of-00002.bin" for i in (1, 2)]
+ folder.mkdir()
+ (folder / "diffusion_pytorch_model.bin.index.json").write_text(
+ json.dumps({"weight_map": {"a": bins[0], "b": bins[1]}})
+ )
+ for shard in bins:
+ (folder / shard).write_bytes(b"x")
+ assert not DiffusionBackend._released_transformer_cached(str(tmp_path))
+ shards = [f"diffusion_pytorch_model-0000{i}-of-00002.safetensors" for i in (1, 2)]
+ _write_index(folder, shards)
+ (folder / shards[0]).write_bytes(b"x")
+ assert not DiffusionBackend._released_transformer_cached(str(tmp_path))
+ (folder / shards[1]).write_bytes(b"x")
+ assert DiffusionBackend._released_transformer_cached(str(tmp_path))
+
+
+def test_shards_scattered_across_revisions_are_not_a_cached_release(monkeypatch, tmp_path):
+ repo_dir = tmp_path / "models--org--repo"
+ shards = [f"diffusion_pytorch_model-0000{i}-of-00002.safetensors" for i in (1, 2)]
+ for rev, shard in (("aaa", shards[0]), ("bbb", shards[1])):
+ folder = repo_dir / "snapshots" / rev / "transformer"
+ _write_index(folder, shards)
+ (folder / shard).write_bytes(b"x")
+ (repo_dir / "snapshots" / "aaa" / "model_index.json").write_text("{}")
+ (repo_dir / "refs").mkdir()
+ (repo_dir / "refs" / "main").write_text("aaa")
+ monkeypatch.setattr(
+ DiffusionBackend, "_hub_cache_repo_dirs", staticmethod(lambda _repo: [repo_dir])
+ )
+ assert not DiffusionBackend._released_transformer_cached("org/repo")
+ (repo_dir / "snapshots" / "aaa" / "transformer" / shards[1]).write_bytes(b"x")
+ assert DiffusionBackend._released_transformer_cached("org/repo")
+
+
+def test_hidreams_pre_cast_fourth_encoder_is_priced_in_the_bf16_plan(monkeypatch):
+ import core.inference.diffusion_te_prequant as teq
+ from core.inference.diffusion_hidream import HIDREAM_LLAMA_BF16_BYTES
+
+ fam = detect_family_for_pick("HiDream-ai/HiDream-I1-Full", None, None)
+ assert fam is not None and fam.name == "hidream-i1"
+ monkeypatch.setattr(teq, "te_prequant_budget_scale", lambda *_a, **_k: 1.0)
+ monkeypatch.setattr(
+ teq,
+ "te_prequant_sources_for_base",
+ lambda _fam, _base, *, te_quant_mode, target, components = (), **_k: (
+ {"text_encoder_4": object()}
+ if te_quant_mode == "fp8" and tuple(components) == ("text_encoder_4",)
+ else {}
+ ),
+ )
+ seen: list = []
+ monkeypatch.setattr(DiffusionBackend, "_plan_memory", lambda _s, *_a, **k: seen.append(k) or k)
+ backend = DiffusionBackend()
+ kwargs = {"kind": "pipeline", "repo_id": "HiDream-ai/HiDream-I1-Full"}
+ backend._bf16_table_plan(_target(), fam, "HiDream-ai/HiDream-I1-Full", None, False, **kwargs)
+ backend._bf16_table_plan(
+ _target(),
+ fam,
+ "HiDream-ai/HiDream-I1-Full",
+ None,
+ False,
+ text_encoder_quant = "fp8",
+ **kwargs,
+ )
+ dense, pre_cast = seen[0]["text_encoder_override_mib"], seen[1]["text_encoder_override_mib"]
+ saved = HIDREAM_LLAMA_BF16_BYTES * (1.0 - teq.TE_PREQUANT_BUDGET_SCALE) / (1024 * 1024)
+ assert abs((dense - pre_cast) - saved) <= 2
+ assert seen[1]["companion_override_mib"] < seen[0]["companion_override_mib"]
+
+
+def test_a_lone_numbered_shard_without_its_index_is_not_a_cached_release(tmp_path):
+ folder = tmp_path / "transformer"
+ folder.mkdir()
+ (folder / "diffusion_pytorch_model-00001-of-00002.safetensors").write_bytes(b"x")
+ assert not DiffusionBackend._released_transformer_cached(str(tmp_path))
+ (folder / "diffusion_pytorch_model.safetensors").write_bytes(b"x")
+ assert DiffusionBackend._released_transformer_cached(str(tmp_path))
+
+
+def test_an_offline_pick_budgets_the_encoder_dense(monkeypatch):
+ import core.inference.diffusion_te_prequant as teq
+
+ backend = _settle_backend(monkeypatch)
+ monkeypatch.setattr(
+ teq,
+ "te_prequant_budget_scale",
+ lambda _fam, *, te_quant_mode, target, base: 0.5 if te_quant_mode == "fp8" else 1.0,
+ )
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_plan_memory",
+ lambda *_a, **k: types.SimpleNamespace(
+ offload_policy = "none"
+ if (k.get("companion_override_mib") or 0) < 6_000
+ or (k.get("transformer_resident_override_mib") or 0) < 80_000
+ else "sequential"
+ ),
+ )
+ _uncompilable(monkeypatch)
+ monkeypatch.setattr(
+ DiffusionBackend, "_released_transformer_cached", staticmethod(lambda _base: True)
+ )
+ assert _settle(backend, text_encoder_quant = "fp8") is None
+ assert _settle(backend, text_encoder_quant = "fp8", local_files_only = True) == "fp8"
+
+
+def test_an_offline_pick_finds_released_shards_under_the_mirror(monkeypatch):
+ backend = _settle_backend(monkeypatch)
+ _offload_when_bf16_sized(monkeypatch, "none")
+ _uncompilable(monkeypatch)
+ monkeypatch.setattr(dmod, "prefer_ungated_mirror", lambda base, *_a, **_k: "unsloth/mirror")
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_released_transformer_cached",
+ staticmethod(lambda base: base == "unsloth/mirror"),
+ )
+ assert _settle(backend, local_files_only = True) is None
+
+
+def test_a_hidream_pipeline_is_repriced_with_its_standalone_encoder(monkeypatch):
+ fam = detect_family_for_pick("HiDream-ai/HiDream-I1-Dev", None, None)
+ assert fam is not None and fam.name == "hidream-i1"
+ as_built = types.SimpleNamespace(offload_policy = "none", estimates = {}, device_memory = None)
+ monkeypatch.setattr(DiffusionBackend, "_resident_sized_plan", lambda _s, plan, *_a, **_k: plan)
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_plan_memory",
+ lambda _s, *_a, **k: types.SimpleNamespace(offload_policy = "group", overrides = k),
+ )
+ repriced = DiffusionBackend()._bf16_resident_plan(
+ as_built,
+ _target(),
+ fam,
+ "HiDream-ai/HiDream-I1-Dev",
+ None,
+ False,
+ kind = "pipeline",
+ repo_id = "HiDream-ai/HiDream-I1-Dev",
+ )
+ assert repriced is not as_built
+ table_gb = 34.2 + 28.8
+ total_mib = (
+ repriced.overrides["transformer_resident_override_mib"]
+ + repriced.overrides["companion_override_mib"]
+ )
+ assert total_mib >= int(table_gb * 1000.0**3 / (1024 * 1024)) - 2
+
+
+def test_the_offline_shard_check_reads_the_root_the_loader_reads(monkeypatch, tmp_path):
+ live, other = tmp_path / "live" / "models--org--repo", tmp_path / "other" / "models--org--repo"
+ for repo_dir, with_transformer in ((live, False), (other, True)):
+ snapshot = repo_dir / "snapshots" / "rev"
+ snapshot.mkdir(parents = True)
+ (snapshot / "model_index.json").write_text("{}")
+ if with_transformer:
+ (snapshot / "transformer").mkdir()
+ (snapshot / "transformer" / "diffusion_pytorch_model.safetensors").write_bytes(b"x")
+ (repo_dir / "refs").mkdir()
+ (repo_dir / "refs" / "main").write_text("rev")
+ monkeypatch.setattr(
+ DiffusionBackend, "_hub_cache_repo_dirs", staticmethod(lambda _repo: [live, other])
+ )
+ assert not DiffusionBackend._released_transformer_cached("org/repo")
+ (live / "snapshots" / "rev" / "model_index.json").unlink()
+ assert DiffusionBackend._released_transformer_cached("org/repo")
+
+
[email protected]("resolved", [{}, {"text_encoder": ("repo", [("te.safetensors", 1)])}])
+def test_pre_cast_sizing_waits_for_a_resolved_encoder_artifact(monkeypatch, resolved):
+ backend, _seen, _fetched = _run_load_backend(monkeypatch, planned = None)
+ monkeypatch.setattr(DiffusionBackend, "_te_prequant_plan_files", lambda *_a, **_k: resolved)
+ heard: dict = {}
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_pipeline_planned_denoiser_scheme",
+ lambda _self, _fam, **k: heard.update(k),
+ )
+ backend._run_load(
+ repo_id = Z_IMAGE_REPO, model_kind = "pipeline", text_encoder_quant = "fp8", _load_token = 1
+ )
+ assert heard["text_encoder_quant"] == ("fp8" if resolved else None)
+
+
+def test_a_dtype_variant_twin_is_not_a_cached_release(tmp_path):
+ folder = tmp_path / "transformer"
+ shards = [f"diffusion_pytorch_model-0000{i}-of-00002.fp16.safetensors" for i in (1, 2)]
+ folder.mkdir()
+ (folder / "diffusion_pytorch_model.fp16.safetensors.index.json").write_text(
+ json.dumps({"weight_map": {"a": shards[0], "b": shards[1]}})
+ )
+ for shard in shards:
+ (folder / shard).write_bytes(b"x")
+ (folder / "diffusion_pytorch_model.fp16.safetensors").write_bytes(b"x")
+ assert not DiffusionBackend._released_transformer_cached(str(tmp_path))
+
+
+def test_an_offline_pick_checks_only_the_repo_the_load_reads(monkeypatch):
+ backend = _settle_backend(monkeypatch)
+ _offload_when_bf16_sized(monkeypatch, "none")
+ _uncompilable(monkeypatch)
+ monkeypatch.setattr(dmod, "prefer_ungated_mirror", lambda base, *_a, **_k: base)
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_released_transformer_cached",
+ staticmethod(lambda base: base == "unsloth/mirror"),
+ )
+ assert _settle(backend, local_files_only = True) == "fp8"
+
+
+def test_an_auxiliary_checkpoint_is_not_a_cached_release(tmp_path):
+ folder = tmp_path / "transformer"
+ folder.mkdir()
+ (folder / "adapter_model.safetensors.index.json").write_text(
+ json.dumps({"weight_map": {"a": "adapter_model-00001-of-00001.safetensors"}})
+ )
+ (folder / "adapter_model-00001-of-00001.safetensors").write_bytes(b"x")
+ (folder / "adapter_model.safetensors").write_bytes(b"x")
+ assert not DiffusionBackend._released_transformer_cached(str(tmp_path))
+
+
[email protected]("resolved", [{}, {"text_encoder": ("repo", [("te.safetensors", 1)])}])
+def test_the_load_hears_whether_the_pre_cast_encoder_resolved(monkeypatch, resolved):
+ backend, seen, _fetched = _run_load_backend(monkeypatch, planned = None)
+ monkeypatch.setattr(DiffusionBackend, "_te_prequant_plan_files", lambda *_a, **_k: resolved)
+ backend._run_load(
+ repo_id = Z_IMAGE_REPO, model_kind = "pipeline", text_encoder_quant = "fp8", _load_token = 1
+ )
+ assert seen["_te_prequant_resolved"] is bool(resolved)
+
+
[email protected]("resolved", [False, True])
+def test_the_load_time_decision_budgets_an_unresolved_encoder_dense(
+ fake_runtime, monkeypatch, resolved
+):
+ backend, _spy = _load_backend(monkeypatch)
+ _uncompilable(monkeypatch)
+ heard: list = []
+ real = DiffusionBackend._bf16_resident_plan
+
+ def _spy_plan(self, plan, *a, **k):
+ heard.append(k.get("text_encoder_quant"))
+ return real(self, plan, *a, **k)
+
+ monkeypatch.setattr(DiffusionBackend, "_bf16_resident_plan", _spy_plan)
+ monkeypatch.setattr(dmod, "resolve_te_quant_request", lambda *_a, **_k: ("fp8", False))
+ _load(
+ backend,
+ _pipeline_prequant_planned = None,
+ _pipeline_prequant_skipped = (),
+ _te_prequant_resolved = resolved,
+ )
+ assert heard == [("fp8" if resolved else None)]
+
+
+def test_the_kept_bf16_weights_are_placed_by_the_plan_that_proved_the_fit(
+ fake_runtime, monkeypatch
+):
+ backend, spy = _load_backend(monkeypatch)
+ _uncompilable(monkeypatch)
+ real_plan = DiffusionBackend._plan_memory
+
+ def _plan(self, *a, **k):
+ plan = real_plan(self, *a, **k)
+ if k.get("transformer_resident_override_mib") is None:
+ plan.offload_policy = "sequential"
+ return plan
+
+ monkeypatch.setattr(DiffusionBackend, "_plan_memory", _plan)
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_resident_sized_plan",
+ lambda _s, plan, *_a, **_k: types.SimpleNamespace(**vars(plan)),
+ )
+ placed: list = []
+ monkeypatch.setattr(
+ dmod,
+ "apply_memory_plan",
+ lambda _pipe, plan, *_a, **_k: placed.append(plan.offload_policy) or ("none", False),
+ )
+ _load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())
+
+ assert spy.quantised == []
+ assert placed == ["none"]
diff --git a/studio/backend/tests/test_diffusion_speed.py b/studio/backend/tests/test_diffusion_speed.py
index 5bbfcbbab04..6fa0f9e5571 100644
--- a/studio/backend/tests/test_diffusion_speed.py
+++ b/studio/backend/tests/test_diffusion_speed.py
@@ -1584,3 +1584,43 @@ def test_auto_dynamic_active_follows_the_torchao_marker():
dit._unsloth_auto_dynamic = True
assert ds_mod.auto_dynamic_active(pipe) is True
assert isinstance(ds_mod.dynamo_graph_count(), int)
+
+
+def test_family_compiles_regionally_reads_the_repeated_blocks_declaration(monkeypatch):
+ from core.inference.diffusion_speed import family_compiles_regionally
+
+ diffusers = types.ModuleType("diffusers")
+ diffusers.Blocked = type("Blocked", (), {"_repeated_blocks": ["Block"]})
+ diffusers.Unblocked = type("Unblocked", (), {"_repeated_blocks": []})
+ diffusers.Undeclared = type("Undeclared", (), {})
+ monkeypatch.setitem(sys.modules, "diffusers", diffusers)
+
+ def fam(cls, denoiser_attr = "transformer"):
+ return types.SimpleNamespace(transformer_class = cls, denoiser_attr = denoiser_attr)
+
+ assert family_compiles_regionally(fam("Blocked")) is True
+ assert family_compiles_regionally(fam("Unblocked")) is False
+ assert family_compiles_regionally(fam("Undeclared")) is True
+ assert family_compiles_regionally(fam("NotInThisDiffusers")) is True
+ assert family_compiles_regionally(fam("Unblocked", denoiser_attr = "unet")) is True
+ assert family_compiles_regionally(fam(None)) is True
+ assert family_compiles_regionally(None) is True
+
+
+def test_family_compiles_regionally_closes_the_dynamo_import_window_first(monkeypatch):
+ import utils.torch_warmup as warmup
+
+ from core.inference.diffusion_speed import family_compiles_regionally
+
+ events: list = []
+ monkeypatch.setattr(warmup, "close_dynamo_import_window", lambda _log: events.append("guard"))
+
+ class _Diffusers(types.ModuleType):
+ def __getattr__(self, name):
+ events.append("probe")
+ raise AttributeError(name)
+
+ monkeypatch.setitem(sys.modules, "diffusers", _Diffusers("diffusers"))
+ fam = types.SimpleNamespace(transformer_class = "Lumina2Transformer2DModel")
+ assert family_compiles_regionally(fam) is True
+ assert events[:2] == ["guard", "probe"]