Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep bf16 for auto precision on image families that cannot compile (#11819)
Daniel Hane5082afd43e2
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex f3732435e07..5c35347921d 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -131,6 +131,7 @@ from .diffusion_speed import (compile_dynamic,compile_eligible,compiled_shapes_are_static,+ family_compiles_regionally,fp16_compile_explicit_only,fp16_unet_offloaded,fresh_compile_count,@@ -192,6 +193,7 @@ from .diffusion_denoiser_prequant import (from .diffusion_prequant import (hosted_fast_accum_conflict,load_prequantized_transformer,+ local_prequant_path_ready,prequant_checkpoint_cached,prequant_unreadable_reason,resolve_prequant_source,@@ -1422,6 +1424,31 @@ def _local_base_transformer_present(base_repo: Optional[str]) -> bool:return False+_UNSHARDED_PIPELINE_WEIGHT_RE = re.compile(+ r"^(?:diffusion_pytorch_model|pytorch_model|model)\.(?:bin|safetensors)$"+)+++def _transformer_folder_complete(folder: Path) -> bool:+ """Complete checkpoint for diffusers' default load: safetensors index first, never a ``.bin`` index."""+ if not folder.is_dir():+ return False+ names = {f.name for f in folder.iterdir() if f.is_file()}+ indexes = [+ n+ for n in names+ if n.endswith(".safetensors.index.json") and _DEFAULT_PIPELINE_WEIGHT_INDEX_RE.match(n)+ ]+ if indexes:+ for index in indexes:+ with open(folder / index, encoding = "utf-8") as fh:+ shards = set((json.load(fh).get("weight_map") or {}).values())+ if shards and shards <= names:+ return True+ return False+ return any(_UNSHARDED_PIPELINE_WEIGHT_RE.match(n) for n in names)++def _dense_candidate_is_prequant(fam: Optional[DiffusionFamily],target: Any,@@ -1527,6 +1554,37 @@ def _pipeline_quant_uncompilable_reason(return None+def _plan_proves_resident(plan: Any) -> bool:+ """Resident AND fits; the planner also stays resident when it cannot read the card."""+ estimates = getattr(plan, "estimates", None) or {}+ budget = estimates.get("safe_device_budget_mib")+ required = estimates.get("resident_required_mib")+ # Unified memory plans 'none' whatever the size (offload frees nothing there), so compare the two as well.+ return (+ getattr(plan, "offload_policy", None) == OFFLOAD_NONE+ and budget is not None+ and required is not None+ and int(required) <= int(budget)+ )+++def _auto_quant_eager_reason(+ fam: Any,+ plan: Any,+ prequant_path: Optional[str] = None,+) -> Optional[str]:+ """Why AUTO keeps bf16 for an uncompilable family, or None; an operator's own checkpoint wins."""+ if not _plan_proves_resident(plan) or family_compiles_regionally(fam):+ return None+ if prequant_path and local_prequant_path_ready(prequant_path):+ return None+ return (+ f"'{getattr(fam, 'name', None)}' cannot be regionally compiled (its transformer declares no "+ "repeated blocks), and an uncompiled quantised transformer runs far slower than the weights "+ "it replaces; they fit on this GPU as they are"+ )++def _clear_exception_frames(exc: BaseException) -> None:"""Release failed-call locals while preserving traceback locations."""errors, seen = [exc], set()@@ -2647,6 +2705,8 @@ class DiffusionBackend:gpu_ordinal = kwargs.get("gpu_ordinal"),repo_id = kwargs["repo_id"],fast_accum = kwargs.get("transformer_quant_fast_accum"),+ text_encoder_quant = te_quant_planned if te_prequant_files else None,+ local_files_only = local_files_only,)if local_files_only and pipeline_planned not in (None, PIPELINE_SEED_DECLINED):# Offline twin of the Hub probe below. The first load left the released shards out of the cache,@@ -2725,6 +2785,7 @@ class DiffusionBackend:))kwargs["_pipeline_prequant_skipped"] = tuple(skipped_transformer_files)+ kwargs["_te_prequant_resolved"] = bool(te_prequant_files)if dit_prequant is not None:expected += int(dit_prequant[2])# Only shards this prefetch staged may be materialised by the dense fallback, so read it off the staged@@ -3019,6 +3080,8 @@ class DiffusionBackend:repo_id: Optional[str] = None,fetch_base: Optional[str] = None,fast_accum: Optional[bool] = None,+ text_encoder_quant: Optional[str] = None,+ local_files_only: bool = False,) -> Optional[str]:"""The scheme an official ``kind == "pipeline"`` pick seeds its denoiser from, settled BEFOREanything is downloaded; None keeps the released bf16 weights and ``PIPELINE_SEED_DECLINED``@@ -3052,6 +3115,42 @@ class DiffusionBackend:is not None):return None+ if auto and not family_compiles_regionally(fam):+ bf16_memory = snapshot_device_memory(target)+ bf16_plan = self._bf16_table_plan(+ target,+ fam,+ base or repo_id or "",+ memory_mode,+ cpu_offload,+ kind = kind,+ repo_id = repo_id,+ fetch_base = fetch_base,+ text_encoder_quant = None if local_files_only else text_encoder_quant,+ device_memory_override = replace(bf16_memory, free_mib = bf16_memory.total_mib),+ )+ # Offline without the released shards, bf16 cannot assemble: a cached seed is the only way.+ if (+ bf16_plan is not None+ and _auto_quant_eager_reason(fam, bf16_plan, transformer_prequant_path)+ is not None+ and not (+ local_files_only+ and not any(+ self._released_transformer_cached(candidate)+ for candidate in dict.fromkeys(+ (+ fetch_base,+ base,+ repo_id,+ prefer_ungated_mirror(base or repo_id or ""),+ )+ )+ if candidate+ )+ )+ ):+ return Nonescheme = select_transformer_quant_scheme(target, mode, family = getattr(fam, "name", None))@@ -3570,6 +3669,7 @@ class DiffusionBackend:gpu_ordinal = load_kwargs.get("gpu_ordinal"),repo_id = repo_id,fast_accum = load_kwargs.get("transformer_quant_fast_accum"),+ text_encoder_quant = te_quant_planned if te_files else None,)if allow_device_probeelse None@@ -4406,6 +4506,7 @@ class DiffusionBackend:# The scheme the plan settled, or PIPELINE_SEED_DECLINED; None for a direct call, which the pull never scoped._pipeline_prequant_planned: Optional[str] = None,_pipeline_prequant_skipped: tuple[str, ...] = (),+ _te_prequant_resolved: bool = True,) -> dict[str, Any]:with self._load_cancel_lock:if _load_token is None:@@ -4624,6 +4725,44 @@ class DiffusionBackend:# visible, and into the refusal so it is actionable.transformer_quant_decline: Optional[str] = Nonetransformer_quant_decline_status = RESOLVED_FELL_BACK+ # A GGUF is excluded while it bakes LoRAs: only the dense build can carry them.+ if (+ transformer_quant == TQ_AUTO+ and pipeline_seed_scheme is None+ and dense_quant_supported_kind(kind)+ and dense_transformer_supported(target)+ and not (kind == "gguf" and _has_active_lora(loras))+ and not family_compiles_regionally(fam)+ and (+ eager_reason := _auto_quant_eager_reason(+ fam,+ eager_plan := self._bf16_resident_plan(+ plan,+ target,+ fam,+ base,+ memory_mode,+ cpu_offload,+ kind = kind,+ repo_id = repo_id,+ fetch_base = fetch_base,+ base_local_dir = _base_local_dir,+ text_encoder_quant = text_encoder_quant+ if _te_prequant_resolved and not local_files_only+ else None,+ ),+ transformer_prequant_path,+ )+ )+ is not None+ ):+ logger.info(+ "diffusion.transformer_quant: auto keeps %s (%s)", kind, eager_reason+ )+ transformer_quant = "off"+ transformer_quant_decline = eager_reason+ # Place the released weights by the plan that proved they fit, not by fp32 shard bytes.+ plan = eager_plannative_scheme = (native_quant_scheme(target, transformer_quant_pinned, family = getattr(fam, "name", None)@@ -6459,6 +6598,137 @@ class DiffusionBackend:pipe.to(device)return pipe+ def _bf16_table_plan(+ self,+ target: DiffusionDeviceTarget,+ fam: DiffusionFamily,+ base: str,+ memory_mode: Optional[str],+ cpu_offload: bool,+ *,+ kind: str,+ repo_id: Optional[str],+ fetch_base: Optional[str] = None,+ base_local_dir: Optional[str] = None,+ text_encoder_quant: Optional[str] = None,+ device_memory_override: Optional[DeviceMemory] = None,+ ) -> Any:+ """Plan priced from the family's bf16-resident table, or None for an unknown family."""+ table = family_bf16_components_gb(fam, base)+ if table is None:+ return None+ text_encoder_gb = table[1]+ try:+ from .diffusion_te_prequant import (+ TE_PREQUANT_BUDGET_SCALE,+ te_prequant_budget_scale,+ te_prequant_sources_for_base,+ )++ te_scale = te_prequant_budget_scale(+ fam, te_quant_mode = text_encoder_quant, target = target, base = base+ )+ text_encoder_gb = table[1] * te_scale+ # HiDream's Llama text_encoder_4 comes pre-cast from its own repo, outside the components the scale probes.+ if (+ te_scale == 1.0+ and getattr(fam, "name", None) == HIDREAM_FAMILY_NAME+ and te_prequant_sources_for_base(+ fam,+ HIDREAM_LLAMA_REPO,+ te_quant_mode = text_encoder_quant,+ target = target,+ components = ("text_encoder_4",),+ standalone_component_bases = {"text_encoder_4": HIDREAM_LLAMA_REPO},+ ).get("text_encoder_4")+ is not None+ ):+ llama_gb = min(HIDREAM_LLAMA_BF16_BYTES / 1000.0**3, table[1])+ text_encoder_gb = table[1] - llama_gb * (1.0 - TE_PREQUANT_BUDGET_SCALE)+ except Exception: # noqa: BLE001 -- sizing aid only; budget the dense encoder+ text_encoder_gb = table[1]+ mib_per_gb = 1000.0**3 / (1024.0 * 1024.0)+ text_encoder_mib = int(text_encoder_gb * mib_per_gb)+ return self._plan_memory(+ target,+ None,+ base,+ fam,+ memory_mode,+ cpu_offload,+ kind = kind,+ repo_id = repo_id,+ fetch_base = fetch_base,+ base_local_dir = base_local_dir,+ transformer_resident_override_mib = int(table[0] * mib_per_gb),+ companion_override_mib = text_encoder_mib + int(table[2] * mib_per_gb),+ text_encoder_override_mib = text_encoder_mib,+ device_memory_override = device_memory_override,+ )++ def _bf16_resident_plan(+ self,+ plan: Any,+ target: DiffusionDeviceTarget,+ fam: DiffusionFamily,+ base: str,+ memory_mode: Optional[str],+ cpu_offload: bool,+ *,+ kind: str,+ repo_id: Optional[str],+ fetch_base: Optional[str] = None,+ base_local_dir: Optional[str] = None,+ text_encoder_quant: Optional[str] = None,+ ) -> Any:+ """``plan`` re-priced when the table lowers it (cached shards wider than bf16).++ HiDream always: no cached plan counts its separately loaded text_encoder_4."""+ if kind != "pipeline":+ return plan+ if (+ getattr(fam, "name", None) != HIDREAM_FAMILY_NAME+ and self._resident_sized_plan(+ plan, fam, base, target, kind, text_encoder_quant = text_encoder_quant+ )+ is plan+ ):+ return plan+ try:+ repriced = self._bf16_table_plan(+ target,+ fam,+ base,+ memory_mode,+ cpu_offload,+ kind = kind,+ repo_id = repo_id,+ fetch_base = fetch_base,+ base_local_dir = base_local_dir,+ text_encoder_quant = text_encoder_quant,+ device_memory_override = getattr(plan, "device_memory", None),+ )+ except Exception: # noqa: BLE001 -- sizing aid only; decide on the plan as built+ return plan+ return plan if repriced is None else repriced++ @staticmethod+ def _released_transformer_cached(base: Optional[str]) -> bool:+ """Complete ``transformer/`` in the snapshot the loader reads; roots in ``_assert_base_repo_accessible`` order."""+ if not base:+ return False+ try:+ local = Path(base).expanduser()+ if local.is_dir():+ return _transformer_folder_complete(local / "transformer")+ for repo_dir in DiffusionBackend._hub_cache_repo_dirs(base):+ snapshot = DiffusionBackend._live_snapshot_dir(repo_dir)+ if snapshot is not None and (snapshot / "model_index.json").is_file():+ return _transformer_folder_complete(snapshot / "transformer")+ return False+ except Exception: # noqa: BLE001 -- unreadable cache: treat the shards as missing+ return False+@staticmethoddef _precast_scaled_companions_mib(candidate: Any,diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.pyindex 3af547b117a..bb64b3fea3c 100644--- a/studio/backend/core/inference/diffusion_speed.py+++ b/studio/backend/core/inference/diffusion_speed.py@@ -283,6 +283,30 @@ def fp16_compile_explicit_only(target: Any) -> bool:return _is_float16(getattr(target, "dtype", None))+def family_compiles_regionally(family: Any) -> bool:+ """False only for an EMPTY ``_repeated_blocks`` (``compile_repeated_blocks`` raises); unknown reads True."""+ if getattr(family, "denoiser_attr", "transformer") != "transformer":+ return True+ name = getattr(family, "transformer_class", None)+ if not isinstance(name, str) or not name:+ return True+ # Called before load_pipeline's guard; `import diffusers` imports torch._dynamo, so wait for the torch warm.+ try:+ from loggers import get_logger+ from utils.torch_warmup import close_dynamo_import_window+ close_dynamo_import_window(get_logger(__name__))+ except Exception: # noqa: BLE001, S110 - optimisation only+ pass+ try:+ import diffusers+ cls = getattr(diffusers, name, None)+ except Exception: # noqa: BLE001 - an unanswerable probe keeps today's behaviour+ return True+ if cls is None or not hasattr(cls, "_repeated_blocks"):+ return True+ return bool(cls._repeated_blocks)++def _is_bfloat16(dtype: Any) -> bool:try:import torchdiff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.pyindex a5ad1fee84c..4f62104cb77 100644--- a/studio/backend/tests/test_diffusion_backend.py+++ b/studio/backend/tests/test_diffusion_backend.py@@ -5248,6 +5248,19 @@ def test_declined_dense_without_loras_still_falls_back_to_gguf(assert backend.status()["transformer_quant"] is None # GGUF-as-is fallback+def test_an_uncompilable_gguf_baking_loras_keeps_the_dense_build(+ fake_runtime, tmp_path, monkeypatch+):+ from core.inference import diffusion as dmod++ backend = DiffusionBackend()+ _decline_dense_quant(backend, monkeypatch, tmp_path)+ monkeypatch.setattr(dmod, "family_compiles_regionally", lambda _fam: False)+ monkeypatch.setattr(dmod, "_plan_proves_resident", lambda _plan: True)+ with pytest.raises(RuntimeError, match = "LoRA adapters could not be applied"):+ _load_m(backend, tmp_path, loras = [("adapter", 1.0)])++class _BakePipe:def __init__(self):self.calls: list = []diff --git a/studio/backend/tests/test_diffusion_pipeline_prequant.py b/studio/backend/tests/test_diffusion_pipeline_prequant.pyindex a6f4f9bb458..6206802e794 100644--- a/studio/backend/tests/test_diffusion_pipeline_prequant.py+++ b/studio/backend/tests/test_diffusion_pipeline_prequant.py@@ -386,9 +386,9 @@ def _settle_backend_walking(monkeypatch, *, artifacts: tuple, candidates: tuple)monkeypatch.setattr(dmod,"denoiser_prequant_source",- lambda fam, scheme, **_k: ("unsloth/Qwen-Image-FP8", f"{scheme}.pt")- if scheme in artifacts- else None,+ lambda fam, scheme, **_k: (+ ("unsloth/Qwen-Image-FP8", f"{scheme}.pt") if scheme in artifacts else None+ ),)monkeypatch.setattr(dmod,@@ -524,9 +524,11 @@ def _run_load_backend(monkeypatch.setattr(DiffusionBackend,"_dit_prequant_plan_source",- lambda *_a, **_k: (PREQUANT_REPO, PREQUANT_FILE, PREQUANT_BYTES)- if verified and planned not in (None, PIPELINE_SEED_DECLINED)- else None,+ lambda *_a, **_k: (+ (PREQUANT_REPO, PREQUANT_FILE, PREQUANT_BYTES)+ if verified and planned not in (None, PIPELINE_SEED_DECLINED)+ else None+ ),)fetched: list = []monkeypatch.setattr(@@ -953,3 +955,601 @@ def test_the_rebuilt_fp8_artifacts_are_listed_for_both_schemes(family, repo):assert fam is not Nonefor scheme in ("fp8", "int8"):assert family_prequant_repo(fam, scheme) == repo+++_MEASURED = {"safe_device_budget_mib": 170_000, "resident_required_mib": 60_000}+++def _uncompilable(monkeypatch, *, measured = True):+ monkeypatch.setattr(dmod, "family_compiles_regionally", lambda _fam: False)+ monkeypatch.setattr(dmod, "family_bf16_components_gb", lambda *_a, **_k: (90.0, 10.0, 0.2))+ inner = DiffusionBackend._plan_memory++ def _plan(self, *a, **k):+ plan = inner(self, *a, **k)+ plan.estimates = dict(_MEASURED) if measured else {}+ return plan++ monkeypatch.setattr(DiffusionBackend, "_plan_memory", _plan)+++def _offload_when_bf16_sized(monkeypatch, bf16_policy):+ monkeypatch.setattr(+ DiffusionBackend,+ "_plan_memory",+ lambda *_a, **k: types.SimpleNamespace(+ offload_policy = bf16_policy+ if (k.get("transformer_resident_override_mib") or 0) > 80_000+ else "none"+ ),+ )+++def test_an_uncompilable_family_keeps_the_released_weights_that_fit(monkeypatch):+ backend = _settle_backend(monkeypatch)+ _offload_when_bf16_sized(monkeypatch, "none")+ _uncompilable(monkeypatch)+ assert _settle(backend) is None+++def test_an_uncompilable_family_still_seeds_when_the_budget_is_unmeasured(monkeypatch):+ backend = _settle_backend(monkeypatch)+ _offload_when_bf16_sized(monkeypatch, "none")+ _uncompilable(monkeypatch, measured = False)+ assert _settle(backend) == "fp8"+++def test_an_uncompilable_family_still_seeds_when_the_released_weights_would_offload(monkeypatch):+ backend = _settle_backend(monkeypatch)+ _offload_when_bf16_sized(monkeypatch, "sequential")+ _uncompilable(monkeypatch)+ assert _settle(backend) == "fp8"+++def test_an_uncompilable_family_still_seeds_an_explicit_scheme(monkeypatch):+ backend = _settle_backend(monkeypatch)+ _offload_when_bf16_sized(monkeypatch, "none")+ _uncompilable(monkeypatch)+ assert _settle(backend, transformer_quant = "fp8") == "fp8"+++def _operator_checkpoint(monkeypatch, tmp_path):+ ckpt = tmp_path / "transformer_fp8.pt"+ ckpt.write_bytes(b"x")+ monkeypatch.setenv(pqmod.ALLOW_LOCAL_PREQUANT_PATH_ENV, str(tmp_path))+ return str(ckpt)+++def test_an_uncompilable_family_still_plans_an_operators_checkpoint(monkeypatch, tmp_path):+ backend = _settle_backend(monkeypatch)+ _offload_when_bf16_sized(monkeypatch, "none")+ _uncompilable(monkeypatch)+ path = _operator_checkpoint(monkeypatch, tmp_path)+ monkeypatch.setattr(pqmod, "local_prequant_scheme", lambda _path: "fp8")+ assert _settle(backend, transformer_prequant_path = path) is not None+ # An unloadable path is no ask: the released weights still win.+ assert _settle(backend, transformer_prequant_path = str(tmp_path / "missing.pt")) is None+++def test_a_compilable_family_seeds_as_before(monkeypatch):+ backend = _settle_backend(monkeypatch)+ monkeypatch.setattr(dmod, "family_compiles_regionally", lambda _fam: True)+ _offload_when_bf16_sized(monkeypatch, "none")+ assert _settle(backend) == "fp8"+++def test_an_uncompilable_family_loads_auto_unquantised(fake_runtime, monkeypatch):+ backend, spy = _load_backend(monkeypatch)+ _uncompilable(monkeypatch)+ status = _load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())++ assert spy.quantised == []+ assert spy.seeds == []+ resolved = status["resolved"]["transformer_quant"]+ assert (resolved["value"], resolved["source"], resolved["status"]) == ("off", "auto", "applied")+ assert "cannot be regionally compiled" in resolved["reason"]+++def test_an_uncompilable_family_loads_an_operators_checkpoint_under_auto(+ fake_runtime, monkeypatch, tmp_path+):+ backend, spy = _load_backend(monkeypatch)+ _uncompilable(monkeypatch)+ status = _load(+ backend,+ transformer_prequant_path = _operator_checkpoint(monkeypatch, tmp_path),+ _pipeline_prequant_planned = None,+ _pipeline_prequant_skipped = (),+ )++ assert spy.quantised == ["auto"]+ assert "cannot be regionally compiled" not in str(status["resolved"]["transformer_quant"])+++def test_a_compilable_family_loads_without_the_bf16_replan(fake_runtime, monkeypatch):+ backend, spy = _load_backend(monkeypatch)+ monkeypatch.setattr(dmod, "family_compiles_regionally", lambda _fam: True)+ replans = []+ monkeypatch.setattr(+ DiffusionBackend,+ "_bf16_resident_plan",+ lambda self, plan, *a, **k: replans.append(1) or plan,+ )+ _load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())++ assert replans == []+ assert spy.quantised == ["auto"]+++def test_an_uncompilable_family_quantises_auto_when_bf16_would_offload(fake_runtime, monkeypatch):+ backend, spy = _load_backend(monkeypatch)+ _uncompilable(monkeypatch)+ real_plan = DiffusionBackend._plan_memory++ def _plan(self, *a, **k):+ plan = real_plan(self, *a, **k)+ if k.get("transformer_resident_override_mib") is None:+ plan.offload_policy = "sequential"+ return plan++ monkeypatch.setattr(DiffusionBackend, "_plan_memory", _plan)+ _load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())++ assert spy.quantised == ["auto"]+++def test_an_uncompilable_family_honours_an_explicit_scheme(fake_runtime, monkeypatch):+ backend, spy = _load_backend(monkeypatch)+ _uncompilable(monkeypatch)+ _load(+ backend,+ transformer_quant = "fp8",+ _pipeline_prequant_planned = None,+ _pipeline_prequant_skipped = (),+ )++ assert spy.quantised == ["fp8"]+++def test_an_uncompilable_family_on_a_host_without_dense_quant_reports_as_before(+ fake_runtime, monkeypatch+):+ backend, spy = _load_backend(monkeypatch)+ _uncompilable(monkeypatch)+ monkeypatch.setattr(dmod, "dense_transformer_supported", lambda _t: False)+ status = _load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())++ assert spy.quantised == []+ assert "regionally compiled" not in status["resolved"]["transformer_quant"]["reason"]+++def test_an_uncompilable_family_quantises_auto_when_the_budget_is_unmeasured(+ fake_runtime, monkeypatch+):+ backend, spy = _load_backend(monkeypatch)+ _uncompilable(monkeypatch, measured = False)+ _load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())++ assert spy.quantised == ["auto"]+++def test_a_resident_plan_whose_requirement_exceeds_the_budget_proves_no_fit():+ fits = types.SimpleNamespace(+ offload_policy = "none",+ estimates = {"safe_device_budget_mib": 60_000, "resident_required_mib": 60_000},+ )+ over = types.SimpleNamespace(+ offload_policy = "none",+ estimates = {"safe_device_budget_mib": 50_000, "resident_required_mib": 60_000},+ )+ assert dmod._plan_proves_resident(fits)+ assert not dmod._plan_proves_resident(over)+++def test_an_uncompilable_family_seeds_when_bf16_overflows_a_unified_pool(monkeypatch):+ backend = _settle_backend(monkeypatch)+ monkeypatch.setattr(dmod, "family_compiles_regionally", lambda _fam: False)+ monkeypatch.setattr(dmod, "family_bf16_components_gb", lambda *_a, **_k: (90.0, 10.0, 0.2))+ monkeypatch.setattr(+ DiffusionBackend,+ "_plan_memory",+ lambda *_a, **_k: types.SimpleNamespace(+ offload_policy = "none",+ estimates = {"safe_device_budget_mib": 50_000, "resident_required_mib": 60_000},+ ),+ )+ assert _settle(backend) == "fp8"+++def test_an_uncompilable_family_prices_a_pre_cast_text_encoder(monkeypatch):+ import core.inference.diffusion_te_prequant as teq++ backend = _settle_backend(monkeypatch)+ monkeypatch.setattr(+ teq,+ "te_prequant_budget_scale",+ lambda _fam, *, te_quant_mode, target, base: 0.5 if te_quant_mode == "fp8" else 1.0,+ )+ monkeypatch.setattr(+ DiffusionBackend,+ "_plan_memory",+ lambda *_a, **k: types.SimpleNamespace(+ offload_policy = "none"+ if (k.get("companion_override_mib") or 0) < 6_000+ or (k.get("transformer_resident_override_mib") or 0) < 80_000+ else "sequential"+ ),+ )+ _uncompilable(monkeypatch)+ assert _settle(backend) == "fp8"+ assert _settle(backend, text_encoder_quant = "fp8") is None+++def test_an_offline_uncompilable_pick_keeps_a_cached_seed_without_released_shards(monkeypatch):+ backend = _settle_backend(monkeypatch)+ _offload_when_bf16_sized(monkeypatch, "none")+ _uncompilable(monkeypatch)+ monkeypatch.setattr(+ DiffusionBackend, "_released_transformer_cached", staticmethod(lambda _base: False)+ )+ assert _settle(backend, local_files_only = True) == "fp8"+ assert _settle(backend) is None+ monkeypatch.setattr(+ DiffusionBackend, "_released_transformer_cached", staticmethod(lambda _base: True)+ )+ assert _settle(backend, local_files_only = True) is None+++def test_released_transformer_cached_reads_the_transformer_folder(tmp_path):+ (tmp_path / "vae").mkdir()+ (tmp_path / "vae" / "diffusion_pytorch_model.safetensors").write_bytes(b"x")+ assert not DiffusionBackend._released_transformer_cached(str(tmp_path))+ (tmp_path / "transformer").mkdir()+ (tmp_path / "transformer" / "diffusion_pytorch_model.safetensors").write_bytes(b"x")+ assert DiffusionBackend._released_transformer_cached(str(tmp_path))+ assert not DiffusionBackend._released_transformer_cached(None)+++def test_the_seed_planner_hears_the_encoder_choice_and_offline_flag(monkeypatch):+ backend, _seen, _fetched = _run_load_backend(monkeypatch, planned = None)+ heard: dict = {}+ monkeypatch.setattr(+ DiffusionBackend,+ "_pipeline_planned_denoiser_scheme",+ lambda _self, _fam, **k: heard.update(k),+ )+ backend._run_load(+ repo_id = Z_IMAGE_REPO, model_kind = "pipeline", local_files_only = True, _load_token = 1+ )+ assert heard["local_files_only"] is True+ assert "text_encoder_quant" in heard+++def test_an_uncompilable_family_keeps_bf16_when_the_resident_table_fits(fake_runtime, monkeypatch):+ backend, spy = _load_backend(monkeypatch)+ _uncompilable(monkeypatch)+ real_plan = DiffusionBackend._plan_memory+ overrides: list = []++ def _plan(self, *a, **k):+ plan = real_plan(self, *a, **k)+ overrides.append(k.get("transformer_resident_override_mib"))+ if k.get("transformer_resident_override_mib") is None:+ plan.offload_policy = "sequential"+ return plan++ monkeypatch.setattr(DiffusionBackend, "_plan_memory", _plan)+ monkeypatch.setattr(+ DiffusionBackend,+ "_resident_sized_plan",+ lambda _s, plan, *_a, **_k: types.SimpleNamespace(**vars(plan)),+ )+ status = _load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())++ assert spy.quantised == []+ assert int(90.0 * 1000.0**3 / (1024.0 * 1024.0)) in overrides+ assert status["resolved"]["transformer_quant"]["value"] == "off"+++def _write_index(folder, shards):+ folder.mkdir(parents = True, exist_ok = True)+ (folder / "diffusion_pytorch_model.safetensors.index.json").write_text(+ json.dumps({"weight_map": {f"w{i}": shard for i, shard in enumerate(shards)}})+ )+++def test_a_partial_sharded_transformer_is_not_a_cached_release(tmp_path):+ shards = [f"diffusion_pytorch_model-0000{i}-of-00002.safetensors" for i in (1, 2)]+ _write_index(tmp_path / "transformer", shards)+ (tmp_path / "transformer" / shards[0]).write_bytes(b"x")+ assert not DiffusionBackend._released_transformer_cached(str(tmp_path))+ (tmp_path / "transformer" / shards[1]).write_bytes(b"x")+ assert DiffusionBackend._released_transformer_cached(str(tmp_path))+++def test_a_complete_bin_set_does_not_cover_a_partial_safetensors_index(tmp_path):+ # Diffusers loads the safetensors index when present and never a .bin index by default.+ folder = tmp_path / "transformer"+ bins = [f"diffusion_pytorch_model-0000{i}-of-00002.bin" for i in (1, 2)]+ folder.mkdir()+ (folder / "diffusion_pytorch_model.bin.index.json").write_text(+ json.dumps({"weight_map": {"a": bins[0], "b": bins[1]}})+ )+ for shard in bins:+ (folder / shard).write_bytes(b"x")+ assert not DiffusionBackend._released_transformer_cached(str(tmp_path))+ shards = [f"diffusion_pytorch_model-0000{i}-of-00002.safetensors" for i in (1, 2)]+ _write_index(folder, shards)+ (folder / shards[0]).write_bytes(b"x")+ assert not DiffusionBackend._released_transformer_cached(str(tmp_path))+ (folder / shards[1]).write_bytes(b"x")+ assert DiffusionBackend._released_transformer_cached(str(tmp_path))+++def test_shards_scattered_across_revisions_are_not_a_cached_release(monkeypatch, tmp_path):+ repo_dir = tmp_path / "models--org--repo"+ shards = [f"diffusion_pytorch_model-0000{i}-of-00002.safetensors" for i in (1, 2)]+ for rev, shard in (("aaa", shards[0]), ("bbb", shards[1])):+ folder = repo_dir / "snapshots" / rev / "transformer"+ _write_index(folder, shards)+ (folder / shard).write_bytes(b"x")+ (repo_dir / "snapshots" / "aaa" / "model_index.json").write_text("{}")+ (repo_dir / "refs").mkdir()+ (repo_dir / "refs" / "main").write_text("aaa")+ monkeypatch.setattr(+ DiffusionBackend, "_hub_cache_repo_dirs", staticmethod(lambda _repo: [repo_dir])+ )+ assert not DiffusionBackend._released_transformer_cached("org/repo")+ (repo_dir / "snapshots" / "aaa" / "transformer" / shards[1]).write_bytes(b"x")+ assert DiffusionBackend._released_transformer_cached("org/repo")+++def test_hidreams_pre_cast_fourth_encoder_is_priced_in_the_bf16_plan(monkeypatch):+ import core.inference.diffusion_te_prequant as teq+ from core.inference.diffusion_hidream import HIDREAM_LLAMA_BF16_BYTES++ fam = detect_family_for_pick("HiDream-ai/HiDream-I1-Full", None, None)+ assert fam is not None and fam.name == "hidream-i1"+ monkeypatch.setattr(teq, "te_prequant_budget_scale", lambda *_a, **_k: 1.0)+ monkeypatch.setattr(+ teq,+ "te_prequant_sources_for_base",+ lambda _fam, _base, *, te_quant_mode, target, components = (), **_k: (+ {"text_encoder_4": object()}+ if te_quant_mode == "fp8" and tuple(components) == ("text_encoder_4",)+ else {}+ ),+ )+ seen: list = []+ monkeypatch.setattr(DiffusionBackend, "_plan_memory", lambda _s, *_a, **k: seen.append(k) or k)+ backend = DiffusionBackend()+ kwargs = {"kind": "pipeline", "repo_id": "HiDream-ai/HiDream-I1-Full"}+ backend._bf16_table_plan(_target(), fam, "HiDream-ai/HiDream-I1-Full", None, False, **kwargs)+ backend._bf16_table_plan(+ _target(),+ fam,+ "HiDream-ai/HiDream-I1-Full",+ None,+ False,+ text_encoder_quant = "fp8",+ **kwargs,+ )+ dense, pre_cast = seen[0]["text_encoder_override_mib"], seen[1]["text_encoder_override_mib"]+ saved = HIDREAM_LLAMA_BF16_BYTES * (1.0 - teq.TE_PREQUANT_BUDGET_SCALE) / (1024 * 1024)+ assert abs((dense - pre_cast) - saved) <= 2+ assert seen[1]["companion_override_mib"] < seen[0]["companion_override_mib"]+++def test_a_lone_numbered_shard_without_its_index_is_not_a_cached_release(tmp_path):+ folder = tmp_path / "transformer"+ folder.mkdir()+ (folder / "diffusion_pytorch_model-00001-of-00002.safetensors").write_bytes(b"x")+ assert not DiffusionBackend._released_transformer_cached(str(tmp_path))+ (folder / "diffusion_pytorch_model.safetensors").write_bytes(b"x")+ assert DiffusionBackend._released_transformer_cached(str(tmp_path))+++def test_an_offline_pick_budgets_the_encoder_dense(monkeypatch):+ import core.inference.diffusion_te_prequant as teq++ backend = _settle_backend(monkeypatch)+ monkeypatch.setattr(+ teq,+ "te_prequant_budget_scale",+ lambda _fam, *, te_quant_mode, target, base: 0.5 if te_quant_mode == "fp8" else 1.0,+ )+ monkeypatch.setattr(+ DiffusionBackend,+ "_plan_memory",+ lambda *_a, **k: types.SimpleNamespace(+ offload_policy = "none"+ if (k.get("companion_override_mib") or 0) < 6_000+ or (k.get("transformer_resident_override_mib") or 0) < 80_000+ else "sequential"+ ),+ )+ _uncompilable(monkeypatch)+ monkeypatch.setattr(+ DiffusionBackend, "_released_transformer_cached", staticmethod(lambda _base: True)+ )+ assert _settle(backend, text_encoder_quant = "fp8") is None+ assert _settle(backend, text_encoder_quant = "fp8", local_files_only = True) == "fp8"+++def test_an_offline_pick_finds_released_shards_under_the_mirror(monkeypatch):+ backend = _settle_backend(monkeypatch)+ _offload_when_bf16_sized(monkeypatch, "none")+ _uncompilable(monkeypatch)+ monkeypatch.setattr(dmod, "prefer_ungated_mirror", lambda base, *_a, **_k: "unsloth/mirror")+ monkeypatch.setattr(+ DiffusionBackend,+ "_released_transformer_cached",+ staticmethod(lambda base: base == "unsloth/mirror"),+ )+ assert _settle(backend, local_files_only = True) is None+++def test_a_hidream_pipeline_is_repriced_with_its_standalone_encoder(monkeypatch):+ fam = detect_family_for_pick("HiDream-ai/HiDream-I1-Dev", None, None)+ assert fam is not None and fam.name == "hidream-i1"+ as_built = types.SimpleNamespace(offload_policy = "none", estimates = {}, device_memory = None)+ monkeypatch.setattr(DiffusionBackend, "_resident_sized_plan", lambda _s, plan, *_a, **_k: plan)+ monkeypatch.setattr(+ DiffusionBackend,+ "_plan_memory",+ lambda _s, *_a, **k: types.SimpleNamespace(offload_policy = "group", overrides = k),+ )+ repriced = DiffusionBackend()._bf16_resident_plan(+ as_built,+ _target(),+ fam,+ "HiDream-ai/HiDream-I1-Dev",+ None,+ False,+ kind = "pipeline",+ repo_id = "HiDream-ai/HiDream-I1-Dev",+ )+ assert repriced is not as_built+ table_gb = 34.2 + 28.8+ total_mib = (+ repriced.overrides["transformer_resident_override_mib"]+ + repriced.overrides["companion_override_mib"]+ )+ assert total_mib >= int(table_gb * 1000.0**3 / (1024 * 1024)) - 2+++def test_the_offline_shard_check_reads_the_root_the_loader_reads(monkeypatch, tmp_path):+ live, other = tmp_path / "live" / "models--org--repo", tmp_path / "other" / "models--org--repo"+ for repo_dir, with_transformer in ((live, False), (other, True)):+ snapshot = repo_dir / "snapshots" / "rev"+ snapshot.mkdir(parents = True)+ (snapshot / "model_index.json").write_text("{}")+ if with_transformer:+ (snapshot / "transformer").mkdir()+ (snapshot / "transformer" / "diffusion_pytorch_model.safetensors").write_bytes(b"x")+ (repo_dir / "refs").mkdir()+ (repo_dir / "refs" / "main").write_text("rev")+ monkeypatch.setattr(+ DiffusionBackend, "_hub_cache_repo_dirs", staticmethod(lambda _repo: [live, other])+ )+ assert not DiffusionBackend._released_transformer_cached("org/repo")+ (live / "snapshots" / "rev" / "model_index.json").unlink()+ assert DiffusionBackend._released_transformer_cached("org/repo")++[email protected]("resolved", [{}, {"text_encoder": ("repo", [("te.safetensors", 1)])}])+def test_pre_cast_sizing_waits_for_a_resolved_encoder_artifact(monkeypatch, resolved):+ backend, _seen, _fetched = _run_load_backend(monkeypatch, planned = None)+ monkeypatch.setattr(DiffusionBackend, "_te_prequant_plan_files", lambda *_a, **_k: resolved)+ heard: dict = {}+ monkeypatch.setattr(+ DiffusionBackend,+ "_pipeline_planned_denoiser_scheme",+ lambda _self, _fam, **k: heard.update(k),+ )+ backend._run_load(+ repo_id = Z_IMAGE_REPO, model_kind = "pipeline", text_encoder_quant = "fp8", _load_token = 1+ )+ assert heard["text_encoder_quant"] == ("fp8" if resolved else None)+++def test_a_dtype_variant_twin_is_not_a_cached_release(tmp_path):+ folder = tmp_path / "transformer"+ shards = [f"diffusion_pytorch_model-0000{i}-of-00002.fp16.safetensors" for i in (1, 2)]+ folder.mkdir()+ (folder / "diffusion_pytorch_model.fp16.safetensors.index.json").write_text(+ json.dumps({"weight_map": {"a": shards[0], "b": shards[1]}})+ )+ for shard in shards:+ (folder / shard).write_bytes(b"x")+ (folder / "diffusion_pytorch_model.fp16.safetensors").write_bytes(b"x")+ assert not DiffusionBackend._released_transformer_cached(str(tmp_path))+++def test_an_offline_pick_checks_only_the_repo_the_load_reads(monkeypatch):+ backend = _settle_backend(monkeypatch)+ _offload_when_bf16_sized(monkeypatch, "none")+ _uncompilable(monkeypatch)+ monkeypatch.setattr(dmod, "prefer_ungated_mirror", lambda base, *_a, **_k: base)+ monkeypatch.setattr(+ DiffusionBackend,+ "_released_transformer_cached",+ staticmethod(lambda base: base == "unsloth/mirror"),+ )+ assert _settle(backend, local_files_only = True) == "fp8"+++def test_an_auxiliary_checkpoint_is_not_a_cached_release(tmp_path):+ folder = tmp_path / "transformer"+ folder.mkdir()+ (folder / "adapter_model.safetensors.index.json").write_text(+ json.dumps({"weight_map": {"a": "adapter_model-00001-of-00001.safetensors"}})+ )+ (folder / "adapter_model-00001-of-00001.safetensors").write_bytes(b"x")+ (folder / "adapter_model.safetensors").write_bytes(b"x")+ assert not DiffusionBackend._released_transformer_cached(str(tmp_path))++[email protected]("resolved", [{}, {"text_encoder": ("repo", [("te.safetensors", 1)])}])+def test_the_load_hears_whether_the_pre_cast_encoder_resolved(monkeypatch, resolved):+ backend, seen, _fetched = _run_load_backend(monkeypatch, planned = None)+ monkeypatch.setattr(DiffusionBackend, "_te_prequant_plan_files", lambda *_a, **_k: resolved)+ backend._run_load(+ repo_id = Z_IMAGE_REPO, model_kind = "pipeline", text_encoder_quant = "fp8", _load_token = 1+ )+ assert seen["_te_prequant_resolved"] is bool(resolved)++[email protected]("resolved", [False, True])+def test_the_load_time_decision_budgets_an_unresolved_encoder_dense(+ fake_runtime, monkeypatch, resolved+):+ backend, _spy = _load_backend(monkeypatch)+ _uncompilable(monkeypatch)+ heard: list = []+ real = DiffusionBackend._bf16_resident_plan++ def _spy_plan(self, plan, *a, **k):+ heard.append(k.get("text_encoder_quant"))+ return real(self, plan, *a, **k)++ monkeypatch.setattr(DiffusionBackend, "_bf16_resident_plan", _spy_plan)+ monkeypatch.setattr(dmod, "resolve_te_quant_request", lambda *_a, **_k: ("fp8", False))+ _load(+ backend,+ _pipeline_prequant_planned = None,+ _pipeline_prequant_skipped = (),+ _te_prequant_resolved = resolved,+ )+ assert heard == [("fp8" if resolved else None)]+++def test_the_kept_bf16_weights_are_placed_by_the_plan_that_proved_the_fit(+ fake_runtime, monkeypatch+):+ backend, spy = _load_backend(monkeypatch)+ _uncompilable(monkeypatch)+ real_plan = DiffusionBackend._plan_memory++ def _plan(self, *a, **k):+ plan = real_plan(self, *a, **k)+ if k.get("transformer_resident_override_mib") is None:+ plan.offload_policy = "sequential"+ return plan++ monkeypatch.setattr(DiffusionBackend, "_plan_memory", _plan)+ monkeypatch.setattr(+ DiffusionBackend,+ "_resident_sized_plan",+ lambda _s, plan, *_a, **_k: types.SimpleNamespace(**vars(plan)),+ )+ placed: list = []+ monkeypatch.setattr(+ dmod,+ "apply_memory_plan",+ lambda _pipe, plan, *_a, **_k: placed.append(plan.offload_policy) or ("none", False),+ )+ _load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())++ assert spy.quantised == []+ assert placed == ["none"]diff --git a/studio/backend/tests/test_diffusion_speed.py b/studio/backend/tests/test_diffusion_speed.pyindex 5bbfcbbab04..6fa0f9e5571 100644--- a/studio/backend/tests/test_diffusion_speed.py+++ b/studio/backend/tests/test_diffusion_speed.py@@ -1584,3 +1584,43 @@ def test_auto_dynamic_active_follows_the_torchao_marker():dit._unsloth_auto_dynamic = Trueassert ds_mod.auto_dynamic_active(pipe) is Trueassert isinstance(ds_mod.dynamo_graph_count(), int)+++def test_family_compiles_regionally_reads_the_repeated_blocks_declaration(monkeypatch):+ from core.inference.diffusion_speed import family_compiles_regionally++ diffusers = types.ModuleType("diffusers")+ diffusers.Blocked = type("Blocked", (), {"_repeated_blocks": ["Block"]})+ diffusers.Unblocked = type("Unblocked", (), {"_repeated_blocks": []})+ diffusers.Undeclared = type("Undeclared", (), {})+ monkeypatch.setitem(sys.modules, "diffusers", diffusers)++ def fam(cls, denoiser_attr = "transformer"):+ return types.SimpleNamespace(transformer_class = cls, denoiser_attr = denoiser_attr)++ assert family_compiles_regionally(fam("Blocked")) is True+ assert family_compiles_regionally(fam("Unblocked")) is False+ assert family_compiles_regionally(fam("Undeclared")) is True+ assert family_compiles_regionally(fam("NotInThisDiffusers")) is True+ assert family_compiles_regionally(fam("Unblocked", denoiser_attr = "unet")) is True+ assert family_compiles_regionally(fam(None)) is True+ assert family_compiles_regionally(None) is True+++def test_family_compiles_regionally_closes_the_dynamo_import_window_first(monkeypatch):+ import utils.torch_warmup as warmup++ from core.inference.diffusion_speed import family_compiles_regionally++ events: list = []+ monkeypatch.setattr(warmup, "close_dynamo_import_window", lambda _log: events.append("guard"))++ class _Diffusers(types.ModuleType):+ def __getattr__(self, name):+ events.append("probe")+ raise AttributeError(name)++ monkeypatch.setitem(sys.modules, "diffusers", _Diffusers("diffusers"))+ fam = types.SimpleNamespace(transformer_class = "Lumina2Transformer2DModel")+ assert family_compiles_regionally(fam) is True+ assert events[:2] == ["guard", "probe"]