Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Hermetic tests for seeding an image PIPELINE pick with its hosted pre-quantized denoiser, which
REPLACES the released ``transformer/`` shards. torch, diffusers and the Hub are stubbed."""
from __future__ import annotations
import contextlib
import json
import sys
import types
import pytest
# A sibling test may put this directory on sys.path and import this file top-level.
try:
from ._torchao_installed import needs_torchao
except ImportError:
from _torchao_installed import needs_torchao
from core.inference import diffusion as dmod
from core.inference import diffusion_prequant as pqmod
from core.inference import diffusion_transformer_quant as tqmod
from core.inference.diffusion import DiffusionBackend
from core.inference.diffusion_auto_policy import DenseQuantEstimate
from core.inference.diffusion_denoiser_prequant import PIPELINE_SEED_DECLINED
from core.inference.diffusion_device import DiffusionDeviceTarget
from core.inference.diffusion_families import ANIMA_BF16_WHEN_RESIDENT, detect_family_for_pick
from core.inference.diffusion_memory import DeviceMemory
from models.inference import DiffusionDownloadPlanResponse
MIB = 1024 * 1024
Z_IMAGE_REPO = "Tongyi-MAI/Z-Image-Turbo"
PREQUANT_REPO = "unsloth/Z-Image-Turbo-FP8"
PREQUANT_FILE = "Z-Image-Turbo-FP8.safetensors"
PREQUANT_BYTES = 6000 * MIB
Z_IMAGE_INDEX = {
"_class_name": "ZImagePipeline",
"transformer": ["diffusers", "ZImageTransformer2DModel"],
"text_encoder": ["transformers", "Qwen3Model"],
"vae": ["diffusers", "AutoencoderKL"],
"tokenizer": ["transformers", "Qwen2Tokenizer"],
"scheduler": ["diffusers", "FlowMatchEulerDiscreteScheduler"],
}
DENOISER_SHARDS = [
"transformer/diffusion_pytorch_model-00001-of-00002.safetensors",
"transformer/diffusion_pytorch_model-00002-of-00002.safetensors",
]
Z_IMAGE_FILES = [
("model_index.json", 100),
("transformer/config.json", 200),
(DENOISER_SHARDS[0], 11_000 * MIB),
(DENOISER_SHARDS[1], 11_000 * MIB),
("text_encoder/model.safetensors", 7_672 * MIB),
("vae/diffusion_pytorch_model.safetensors", 160 * MIB),
("tokenizer/tokenizer_config.json", 300),
("scheduler/scheduler_config.json", 300),
]
DENOISER_BYTES = 22_000 * MIB
ALL_BYTES = sum(size for _name, size in Z_IMAGE_FILES)
@pytest.fixture(autouse = True)
def _unmeasured_torchao(monkeypatch):
"""Pin "no measured torchao" so the installed release does not decide the offload tiers."""
from core.inference import diffusion_memory
monkeypatch.setattr(diffusion_memory, "_installed_torchao_version", lambda: None)
# pin Studio's diffusers version so measured-torchao tests ignore the runner
monkeypatch.setattr(diffusion_memory, "_installed_diffusers_version", lambda: (0, 40))
@pytest.fixture(autouse = True)
def _safetensors_readable_regardless_of_the_installed_torchao(monkeypatch):
"""Pin the torchao floor so planning tests ignore the installed release."""
import core.inference.prequant_safetensors as prequant_safetensors
monkeypatch.setattr(prequant_safetensors, "_torchao_version", lambda: None)
def _family():
fam = detect_family_for_pick(Z_IMAGE_REPO, None, None)
assert fam is not None and fam.name == "z-image"
return fam
def _target(*, device = "cuda", ordinal = None) -> DiffusionDeviceTarget:
return DiffusionDeviceTarget(
device = device,
dtype = "bfloat16",
backend = device,
vendor = "nvidia",
supports_model_cpu_offload = True,
supports_default_torch_compile = True,
supports_pinned_transfer = True,
ordinal = ordinal,
)
def _estimate(prequant = True) -> DenseQuantEstimate:
return DenseQuantEstimate(
scheme = "fp8",
steady_transformer_mib = 11_500,
transient_transformer_mib = 11_500 if prequant else 23_000,
companions_mib = 7_500,
prequant = prequant,
download_transformer_mib = 23_000,
text_encoders_mib = 7_320,
)
@pytest.fixture
def hub(monkeypatch, tmp_path):
"""The pipeline repo plus the hosted checkpoint's repo, as ``model_info`` sees them."""
repos = {
Z_IMAGE_REPO: Z_IMAGE_FILES,
PREQUANT_REPO: [(PREQUANT_FILE, PREQUANT_BYTES), ("README.md", 10)],
}
manifest = tmp_path / "model_index.json"
manifest.write_text(json.dumps(Z_IMAGE_INDEX), encoding = "utf-8")
info_calls: list = []
class _Api:
def __init__(self, *_a, **_k) -> None:
pass
def model_info(self, repo_id, **_kwargs):
info_calls.append(repo_id)
return types.SimpleNamespace(
siblings = [
types.SimpleNamespace(rfilename = name, size = size)
for name, size in repos[repo_id]
],
sha = "c" * 40,
)
monkeypatch.setattr("huggingface_hub.HfApi", _Api)
monkeypatch.setattr(dmod, "prefer_ungated_mirror", lambda base, *_a, **_k: base)
monkeypatch.setattr(
"huggingface_hub.hf_hub_download", lambda repo_id, filename, **kw: str(manifest)
)
return info_calls
def _estimate_bytes(**overrides):
kwargs = {
"kind": "pipeline",
"pipeline_components": None,
"single_file_is_pipeline": False,
**overrides,
}
return DiffusionBackend._estimate_download_bytes(
Z_IMAGE_REPO, None, Z_IMAGE_REPO, None, **kwargs
)
def test_the_plan_drops_the_released_denoiser_shards_and_keeps_its_config(hub):
"""The pull drops the denoiser shards but keeps the config the seeding loader meta-inits from."""
skipped: list[str] = []
resident: list = []
total, files = _estimate_bytes(
skip_transformer_weights = True,
skipped_files_out = skipped,
resident_file_sizes_out = resident,
)
assert "transformer/config.json" in files
assert not [f for f in files if f in DENOISER_SHARDS]
assert skipped == DENOISER_SHARDS
assert total == ALL_BYTES - DENOISER_BYTES
assert not [name for name, _size in resident if name.startswith("transformer/")]
def test_the_plan_keeps_the_shards_when_nothing_is_seeded(hub):
skipped: list[str] = []
total, files = _estimate_bytes(skipped_files_out = skipped)
assert all(shard in files for shard in DENOISER_SHARDS)
assert skipped == []
assert total == ALL_BYTES
def _plan_backend(
monkeypatch,
*,
planned,
mismatch = None,
):
backend = DiffusionBackend()
monkeypatch.setattr(backend, "_target_for_ordinal", lambda *_a, **_k: _target())
monkeypatch.setattr(dmod, "detect_family_for_pick", lambda *_a, **_k: _family())
monkeypatch.setattr(dmod, "prefer_ungated_mirror", lambda base, *_a, **_k: base)
monkeypatch.setattr(dmod, "_assert_base_repo_accessible", lambda *_a, **_k: None)
monkeypatch.setattr(DiffusionBackend, "_te_prequant_plan_files", lambda *_a, **_k: {})
monkeypatch.setattr(dmod, "flux2_pick_mismatch", lambda *_a, **_k: mismatch)
monkeypatch.setattr(dmod, "speech_pick_refusal", lambda *_a, **_k: None)
monkeypatch.setattr(
DiffusionBackend, "_pipeline_planned_denoiser_scheme", lambda *_a, **_k: planned
)
monkeypatch.setattr(
DiffusionBackend, "_files_already_cached", staticmethod(lambda *_a, **_k: set())
)
monkeypatch.setattr(
DiffusionBackend, "_hub_file_is_cached", staticmethod(lambda *_a, **_k: False)
)
monkeypatch.setattr(DiffusionBackend, "declared_footprint_shortfall", lambda *_a, **_k: None)
return backend
@needs_torchao
def test_the_plan_counts_and_stages_the_hosted_checkpoint(monkeypatch, hub):
backend = _plan_backend(monkeypatch, planned = "fp8")
plan = backend.download_plan(Z_IMAGE_REPO, model_kind = "pipeline")
entries = {entry["repo_id"]: entry for entry in plan["entries"]}
assert entries[PREQUANT_REPO]["files"] == [PREQUANT_FILE]
assert entries[PREQUANT_REPO]["bytes"] == PREQUANT_BYTES
assert not [f for f in entries[Z_IMAGE_REPO]["files"] if f in DENOISER_SHARDS]
assert "transformer/config.json" in entries[Z_IMAGE_REPO]["files"]
assert plan["required_bytes"] == ALL_BYTES - DENOISER_BYTES + PREQUANT_BYTES
assert entries[Z_IMAGE_REPO]["checkpoint"] is True
assert entries[PREQUANT_REPO]["checkpoint"] is True
def test_an_unplanned_pipeline_pick_plans_exactly_as_before(monkeypatch, hub):
"""With no hosted artifact the released shards stay in the pull and the plan is unchanged."""
backend = _plan_backend(monkeypatch, planned = None)
plan = backend.download_plan(Z_IMAGE_REPO, model_kind = "pipeline")
entries = {entry["repo_id"]: entry for entry in plan["entries"]}
assert PREQUANT_REPO not in entries
assert all(shard in entries[Z_IMAGE_REPO]["files"] for shard in DENOISER_SHARDS)
assert plan["required_bytes"] == ALL_BYTES
def test_a_declined_plan_keeps_the_released_shards(monkeypatch, hub):
backend = _plan_backend(monkeypatch, planned = PIPELINE_SEED_DECLINED)
plan = backend.download_plan(Z_IMAGE_REPO, model_kind = "pipeline")
entries = {entry["repo_id"]: entry for entry in plan["entries"]}
assert PREQUANT_REPO not in entries
assert all(shard in entries[Z_IMAGE_REPO]["files"] for shard in DENOISER_SHARDS)
@needs_torchao
def test_the_artifact_carries_the_denoiser_share_of_the_unified_memory_verdict(monkeypatch, hub):
"""The unified-memory refusal prices the denoiser from the artifact, not the dropped shards."""
backend = _plan_backend(monkeypatch, planned = "fp8")
seen: dict = {}
def _shortfall(_self, *_a, **kwargs):
seen.update(kwargs)
return None
monkeypatch.setattr(DiffusionBackend, "declared_footprint_shortfall", _shortfall)
backend.download_plan(Z_IMAGE_REPO, model_kind = "pipeline")
assert seen["prequant_bytes"] == PREQUANT_BYTES
assert not [name for name, _s in seen["declared_files"] if name.startswith("transformer/")]
def test_the_hosted_vae_carries_its_share_of_the_unified_memory_verdict(monkeypatch, hub):
"""A hosted VAE that replaces the dropped vae/ shards is priced in the memory verdict."""
backend = _plan_backend(monkeypatch, planned = None)
hosted = {"vae": ("unsloth/hosted-vae", [("vae.safetensors", 675)])}
monkeypatch.setattr(DiffusionBackend, "_hosted_vae_plan_files", lambda *_a, **_k: hosted)
seen: dict = {}
def _shortfall(_self, *_a, **kwargs):
seen.update(kwargs)
return None
monkeypatch.setattr(DiffusionBackend, "declared_footprint_shortfall", _shortfall)
backend.download_plan(Z_IMAGE_REPO, model_kind = "pipeline")
assert seen["prequant_bytes"] == 675
def test_a_gguf_pick_still_never_downloads_a_second_denoiser(monkeypatch):
"""A GGUF pick never sizes an uncached hosted checkpoint: there it is a SECOND denoiser."""
backend = DiffusionBackend()
monkeypatch.setattr(backend, "_target_for_ordinal", lambda *_a, **_k: _target())
monkeypatch.setattr(dmod, "_uncached_prequant_repo", lambda *_a, **_k: PREQUANT_REPO)
def _never(*_a, **_k):
raise AssertionError("a GGUF pick sized an uncached hosted checkpoint")
monkeypatch.setattr(DiffusionBackend, "_prequant_source_hub_entry", staticmethod(_never))
assert (
backend._dit_prequant_plan_source(
_family(), "gguf", None, {"base_repo": Z_IMAGE_REPO, "transformer_quant": None}
)
is None
)
def test_a_planned_pipeline_pick_is_sized_from_the_hub(monkeypatch, hub):
backend = DiffusionBackend()
monkeypatch.setattr(backend, "_target_for_ordinal", lambda *_a, **_k: _target())
monkeypatch.setattr(
pqmod, "restricted_prequant_load_supported", lambda _scheme, filename = None: True
)
assert backend._dit_prequant_plan_source(
_family(),
"pipeline",
None,
{"base_repo": Z_IMAGE_REPO, "_pipeline_prequant_planned": "fp8"},
) == (PREQUANT_REPO, PREQUANT_FILE, PREQUANT_BYTES)
def test_a_pipeline_pick_with_no_planned_scheme_sizes_nothing(monkeypatch, hub):
backend = DiffusionBackend()
for planned in (None, PIPELINE_SEED_DECLINED):
assert (
backend._dit_prequant_plan_source(
_family(),
"pipeline",
None,
{"base_repo": Z_IMAGE_REPO, "_pipeline_prequant_planned": planned},
)
is None
)
def _settle_backend(
monkeypatch,
*,
offload = "none",
scheme = "fp8",
candidate = None,
):
backend = DiffusionBackend()
monkeypatch.setattr(backend, "_target_for_ordinal", lambda *_a, **_k: _target())
monkeypatch.setattr(dmod, "dense_transformer_supported", lambda _t: True)
monkeypatch.setattr(dmod, "_pipeline_quant_uncompilable_reason", lambda *_a, **_k: None)
monkeypatch.setattr(
dmod, "select_transformer_quant_scheme", lambda target, mode, family = None, **_k: scheme
)
monkeypatch.setattr(
pqmod, "restricted_prequant_load_supported", lambda _scheme, filename = None: True
)
monkeypatch.setattr(
dmod,
"resolve_dense_quant_candidate",
lambda **_k: _estimate() if candidate is None else candidate,
)
monkeypatch.setattr(
dmod,
"snapshot_device_memory",
lambda _t: DeviceMemory("cuda", "cuda", "dedicated", 1_000, 180_000),
)
monkeypatch.setattr(
DiffusionBackend,
"_plan_memory",
lambda *_a, **_k: types.SimpleNamespace(offload_policy = offload),
)
return backend
def _settle(backend, **overrides):
kwargs = {
"base": Z_IMAGE_REPO,
"kind": "pipeline",
"transformer_quant": None,
"speed_mode": None,
"repo_id": Z_IMAGE_REPO,
**overrides,
}
return backend._pipeline_planned_denoiser_scheme(_family(), **kwargs)
def test_an_unset_precision_defaults_to_the_hosted_checkpoint(monkeypatch):
assert _settle(_settle_backend(monkeypatch)) == "fp8"
def test_an_ampere_host_defaults_to_the_hosted_int8_checkpoint(monkeypatch):
assert _settle(_settle_backend(monkeypatch, scheme = "int8")) == "int8"
@pytest.mark.parametrize("quant", ["none", "off"])
def test_precision_off_keeps_the_released_weights(monkeypatch, quant):
assert _settle(_settle_backend(monkeypatch), transformer_quant = quant) is None
def test_speed_off_keeps_the_released_weights(monkeypatch):
"""Speed=off keeps the released weights: it is a bit-exact contract."""
assert _settle(_settle_backend(monkeypatch), speed_mode = "off") is None
def test_an_explicit_scheme_under_speed_off_still_seeds(monkeypatch):
"""Speed=off only silences an AUTO precision. An explicit scheme is still quantized and upgrades
the speed to `default`, so it must seed the hosted checkpoint rather than pull the bf16 shards."""
backend = _settle_backend(monkeypatch)
assert _settle(backend, transformer_quant = "fp8", speed_mode = "off") == "fp8"
def test_a_baked_lora_keeps_the_dense_path(monkeypatch):
"""A LoRA bake keeps the dense path; an all-zero list bakes nothing and is unaffected."""
backend = _settle_backend(monkeypatch)
assert _settle(backend, loras = [("adapter", 0.8)]) is None
assert _settle(backend, loras = [("adapter", 0.0)]) == "fp8"
def test_balanced_keeps_the_released_weights(monkeypatch):
"""balanced streams through group-offload hooks the preflight refuses torchao weights under."""
assert _settle(_settle_backend(monkeypatch), memory_mode = "balanced") is None
def _forced_offload_backend(
monkeypatch,
*,
scheme = "fp8",
survives = True,
native = None,
):
"""low_vram's whole-module plan: the denoiser is offloaded, and ``survives`` says whether its torchao weights
survive that placement. ``native`` is the scheme the loader would serve torchao-free under offload."""
backend = _settle_backend(monkeypatch, offload = "model", scheme = scheme)
monkeypatch.setattr(dmod, "torchao_survives_plan", lambda *_a, **_k: survives)
monkeypatch.setattr(
dmod,
"native_quant_scheme",
lambda target, requested, family = None, offload = False: (
requested if offload and requested == native else None
),
)
return backend
@pytest.mark.parametrize("request_kwargs", [{"memory_mode": "low_vram"}, {"cpu_offload": True}])
def test_a_whole_module_offload_request_seeds_the_hosted_checkpoint(monkeypatch, request_kwargs):
"""low_vram (and the legacy flag) offload whole modules, which the runtime fp8 quantise already runs torchao
under; the artifact is seeded on the host instead of quantising the released bf16 shards."""
assert _settle(_forced_offload_backend(monkeypatch), **request_kwargs) == "fp8"
def test_a_whole_module_offload_request_never_seeds_a_torchao_free_rung(monkeypatch):
"""A PINNED int8 under offload runs torchao-free (native kernels); a torchao artifact cannot feed that path."""
backend = _forced_offload_backend(monkeypatch, scheme = "int8", native = "int8")
assert _settle(backend, memory_mode = "low_vram", transformer_quant = "int8") is None
def test_auto_under_a_whole_module_offload_seeds_the_winner_it_quantises(monkeypatch):
"""Auto under low_vram uses torchao, not native, so the winner's artifact is seeded."""
backend = _forced_offload_backend(monkeypatch, scheme = "int8", native = "int8")
assert _settle(backend, memory_mode = "low_vram") == "int8"
def test_auto_under_a_whole_module_offload_never_walks_to_a_lower_rung(monkeypatch):
"""Under a forced offload a lower rung would change the scheme the runtime picks, so a winner whose artifact does
not survive the placement declines instead of seeding the next rung."""
from core.inference import diffusion_transformer_quant as tq
backend = _forced_offload_backend(monkeypatch, scheme = "int8")
monkeypatch.setattr(tq, "auto_scheme_candidates", lambda *_a, **_k: ("int8", "fp8"))
monkeypatch.setattr(dmod, "torchao_survives_plan", lambda _plan, rung, **_k: rung == "fp8")
assert _settle(backend, memory_mode = "low_vram") == PIPELINE_SEED_DECLINED
def test_a_whole_module_offload_request_declines_when_torchao_does_not_survive(monkeypatch):
backend = _forced_offload_backend(monkeypatch, survives = False)
assert _settle(backend, memory_mode = "low_vram") == PIPELINE_SEED_DECLINED
def test_an_artifact_sized_plan_that_still_offloads_declines(monkeypatch):
"""An artifact-sized plan that still offloads pins a decline: torchao rejects offload hooks."""
backend = _settle_backend(monkeypatch, offload = "sequential")
assert _settle(backend) == PIPELINE_SEED_DECLINED
def _settle_backend_walking(monkeypatch, *, artifacts: tuple, candidates: tuple):
"""Memory verdict depends on the rung: int8 (31 GB) offloads, fp8 (19 GB) stays resident."""
from core.inference import diffusion_transformer_quant as tq
backend = _settle_backend(monkeypatch, scheme = candidates[0])
monkeypatch.setattr(tq, "auto_scheme_candidates", lambda target, family = None, **_k: candidates)
monkeypatch.setattr(
dmod,
"denoiser_prequant_source",
lambda fam, scheme, **_k: (
("unsloth/Qwen-Image-FP8", f"{scheme}.pt") if scheme in artifacts else None
),
)
monkeypatch.setattr(
dmod,
"resolve_dense_quant_candidate",
lambda **k: DenseQuantEstimate(
scheme = k["requested"],
steady_transformer_mib = 31_000 if k["requested"] == "int8" else 19_000,
transient_transformer_mib = 0,
companions_mib = 7_500,
prequant = True,
download_transformer_mib = 0,
text_encoders_mib = 7_320,
),
)
monkeypatch.setattr(
DiffusionBackend,
"_plan_memory",
lambda *_a, **k: types.SimpleNamespace(
offload_policy = "sequential"
if k["transformer_resident_override_mib"] >= 31_000
else "none"
),
)
return backend
def test_an_artifact_too_large_for_the_card_yields_to_the_next_hosted_rung(monkeypatch):
"""int8 leads the ladder, but Qwen-Image's int8 file is 12 GB larger than its fp8 one: where the
int8-sized plan offloads, auto seeds the fp8 artifact instead of pinning a decline."""
backend = _settle_backend_walking(
monkeypatch, artifacts = ("int8", "fp8"), candidates = ("int8", "fp8")
)
assert _settle(backend) == "fp8"
def test_a_walk_with_no_resident_rung_declines(monkeypatch):
"""Every hosted rung offloads: the decline is pinned so plan and load agree."""
backend = _settle_backend_walking(monkeypatch, artifacts = ("int8",), candidates = ("int8", "fp8"))
assert _settle(backend) == PIPELINE_SEED_DECLINED
def test_auto_planning_passes_the_base_and_prequant_probe(monkeypatch):
"""Without the base and a checkpoint probe AUTO drops nvfp4, so the plan must ask like the load."""
from core.inference import diffusion_transformer_quant as tq
backend = _settle_backend(monkeypatch)
seen: dict = {}
def _select(
target,
mode,
family = None,
*,
base_repo = None,
has_prequant = None,
**_k,
):
seen["select"] = base_repo
if base_repo == Z_IMAGE_REPO and has_prequant is not None and has_prequant("nvfp4"):
return "nvfp4"
return "mxfp8"
def _candidates(
target,
family = None,
*,
base_repo = None,
has_prequant = None,
**_k,
):
seen["candidates"] = base_repo
head = ("nvfp4",) if has_prequant is not None and has_prequant("nvfp4") else ()
return head + ("mxfp8",)
monkeypatch.setattr(dmod, "select_transformer_quant_scheme", _select)
monkeypatch.setattr(tq, "auto_scheme_candidates", _candidates)
monkeypatch.setattr(
dmod,
"usable_prequant_source",
lambda fam, scheme, **_k: object() if scheme == "nvfp4" else None,
)
monkeypatch.setattr(
dmod,
"denoiser_prequant_source",
lambda fam, scheme, **_k: ("unsloth/Z-Image-Turbo-NVFP4", "z.safetensors")
if scheme == "nvfp4"
else None,
)
assert _settle(backend) == "nvfp4"
assert seen == {"select": Z_IMAGE_REPO, "candidates": Z_IMAGE_REPO}
def test_an_explicit_scheme_is_never_swapped_for_a_lower_rung(monkeypatch):
"""An explicit int8 that offloads declines; auto's walk is not offered to an explicit request."""
backend = _settle_backend_walking(
monkeypatch, artifacts = ("int8", "fp8"), candidates = ("int8", "fp8")
)
assert _settle(backend, transformer_quant = "int8") == PIPELINE_SEED_DECLINED
def test_a_family_with_no_hosted_artifact_falls_through(monkeypatch):
"""A scheme with no hosted artifact falls through to the in-memory quantise."""
assert _settle(_settle_backend(monkeypatch, scheme = "mxfp8")) is None
def test_a_base_with_no_hosted_artifact_falls_through(monkeypatch):
"""An excluded base (the undistilled Z-Image) quantises its own weights."""
assert _settle(_settle_backend(monkeypatch), base = "Tongyi-MAI/Z-Image") is None
def test_a_forced_fp8_accumulate_the_artifact_cannot_bake_keeps_the_released_weights(monkeypatch):
"""The hosted fp8 checkpoints bake fast accumulate, and ``_validate_checkpoint`` refuses a baked
value differing from a FORCED one, so seeding drops the shards for a checkpoint the load rejects."""
backend = _settle_backend(monkeypatch)
assert _settle(backend, transformer_quant = "fp8", fast_accum = False) is None
assert _settle(backend, transformer_quant = "fp8", fast_accum = True) == "fp8"
assert _settle(backend, transformer_quant = "fp8") == "fp8"
def test_a_forced_fp8_accumulate_also_rules_out_a_local_prequant_cache_entry(monkeypatch, tmp_path):
"""A cache entry bakes in auto accumulate, so a forced different value must not seed it."""
from core.inference import diffusion_prequant_cache as cache
root = tmp_path / "prequant_cache"
monkeypatch.setenv(cache.ENV_DIR, str(root))
entry = str(root / ("e" * 32) / cache.FILE_NAME)
operator_file = str(tmp_path / "mine" / "fp8.safetensors")
backend = _settle_backend(monkeypatch)
for location, forced_off in ((entry, None), (operator_file, "fp8")):
monkeypatch.setattr(
dmod,
"denoiser_prequant_source",
lambda fam, scheme, _loc = location, **_k: pqmod.PrequantSource(
kind = "path", location = _loc, filename = None
),
)
assert _settle(backend, transformer_quant = "fp8", fast_accum = False) == forced_off
assert _settle(backend, transformer_quant = "fp8", fast_accum = True) == "fp8"
def test_a_forced_accumulate_never_blocks_a_scheme_that_bakes_none(monkeypatch):
"""Only fp8 records ``fast_accum``, so the int8 artifact is seeded whatever the caller forces."""
backend = _settle_backend(monkeypatch, scheme = "int8")
assert _settle(backend, transformer_quant = "int8", fast_accum = False) == "int8"
def test_a_gguf_pick_is_never_settled_here(monkeypatch):
assert _settle(_settle_backend(monkeypatch), kind = "gguf") is None
@pytest.mark.parametrize("family", ["krea-2", "ideogram-4"])
def test_a_per_component_assembler_is_never_seeded(monkeypatch, family):
"""A per-component assembler is never seeded: it never sees ``pipe_kwargs``."""
backend = _settle_backend(monkeypatch)
fam = types.SimpleNamespace(name = family, base_repo = "x/y")
assert (
backend._pipeline_planned_denoiser_scheme(
fam,
base = "x/y",
kind = "pipeline",
transformer_quant = None,
speed_mode = None,
)
is None
)
def test_an_uncompilable_host_keeps_the_released_weights(monkeypatch):
"""An uncompilable host keeps the released weights, since the loader would refuse the seed."""
backend = _settle_backend(monkeypatch)
monkeypatch.setattr(dmod, "_pipeline_quant_uncompilable_reason", lambda *_a, **_k: "no triton")
assert _settle(backend) is None
def test_an_unanswerable_probe_keeps_the_released_weights(monkeypatch):
backend = _settle_backend(monkeypatch)
def _boom(*_a, **_k):
raise RuntimeError("driver went away")
monkeypatch.setattr(dmod, "select_transformer_quant_scheme", _boom)
assert _settle(backend) is None
def _run_load_backend(
monkeypatch,
*,
planned,
verified = True,
):
backend = DiffusionBackend()
backend._load_token = 1
backend._loading = dmod._LoadingState(repo_id = Z_IMAGE_REPO, base_repo = Z_IMAGE_REPO)
monkeypatch.setattr(dmod, "detect_family_for_pick", lambda *_a, **_k: _family())
monkeypatch.setattr(dmod, "prefer_ungated_mirror", lambda base, *_a, **_k: base)
monkeypatch.setattr(dmod, "_assert_base_repo_accessible", lambda *_a, **_k: None)
monkeypatch.setattr(dmod, "assert_flux2_pick_compatible", lambda *_a, **_k: None)
monkeypatch.setattr(dmod, "assert_pick_is_not_speech", lambda *_a, **_k: None)
monkeypatch.setattr(dmod, "_local_base_transformer_present", lambda *_a, **_k: False)
monkeypatch.setattr(DiffusionBackend, "_te_prequant_plan_files", lambda *_a, **_k: {})
monkeypatch.setattr(DiffusionBackend, "declared_footprint_shortfall", lambda *_a, **_k: None)
monkeypatch.setattr(
DiffusionBackend, "_pipeline_planned_denoiser_scheme", lambda *_a, **_k: planned
)
monkeypatch.setattr(
DiffusionBackend,
"_dit_prequant_plan_source",
lambda *_a, **_k: (
(PREQUANT_REPO, PREQUANT_FILE, PREQUANT_BYTES)
if verified and planned not in (None, PIPELINE_SEED_DECLINED)
else None
),
)
fetched: list = []
monkeypatch.setattr(
DiffusionBackend,
"_fetch_denoiser_prequant",
lambda _self, entry, *_a, **_k: fetched.append(entry),
)
monkeypatch.setattr(DiffusionBackend, "_prefetch_files", lambda *_a, **_k: None)
seen: dict = {}
monkeypatch.setattr(backend, "load_pipeline", lambda **kwargs: seen.update(kwargs))
return backend, seen, fetched
def test_the_load_is_pinned_to_the_plan_that_scoped_the_pull(monkeypatch, hub):
backend, seen, fetched = _run_load_backend(monkeypatch, planned = "fp8")
backend._run_load(repo_id = Z_IMAGE_REPO, model_kind = "pipeline", _load_token = 1)
assert backend._loading is None, getattr(backend._loading, "error", None)
assert seen["_pipeline_prequant_planned"] == "fp8"
assert seen["_pipeline_prequant_skipped"] == tuple(DENOISER_SHARDS)
assert fetched == [(PREQUANT_REPO, PREQUANT_FILE, PREQUANT_BYTES)]
assert PREQUANT_REPO in backend.loading_repo_ids() or backend._loading is None
def test_an_artifact_that_does_not_resolve_keeps_the_released_shards(monkeypatch, hub):
"""An artifact that does not resolve on the Hub keeps the released shards in the pull."""
backend, seen, fetched = _run_load_backend(monkeypatch, planned = "fp8", verified = False)
backend._run_load(repo_id = Z_IMAGE_REPO, model_kind = "pipeline", _load_token = 1)
assert seen["_pipeline_prequant_planned"] is None
assert seen["_pipeline_prequant_skipped"] == ()
assert fetched == []
def test_a_local_checkpoint_is_still_seeded_on_an_online_load(monkeypatch, hub):
"""An operator's own checkpoint has no Hub entry, so the plan drops no shards for it; the seed
must still survive the pull, or the load quantises bf16 in memory and ignores the given file."""
backend, seen, fetched = _run_load_backend(monkeypatch, planned = "fp8", verified = False)
monkeypatch.setattr(dmod, "denoiser_prequant_cached", lambda *_a, **_k: True)
backend._run_load(
repo_id = Z_IMAGE_REPO,
model_kind = "pipeline",
transformer_prequant_path = "/models/z-image-fp8.pt",
_load_token = 1,
)
assert seen["_pipeline_prequant_planned"] == "fp8"
assert seen["_pipeline_prequant_skipped"] == ()
assert fetched == []
def test_a_local_path_the_loader_would_refuse_keeps_the_released_shards(monkeypatch, hub):
"""A seed needs its artifact; an unresolvable path keeps bf16 shards rather than a seed."""
backend, seen, _fetched = _run_load_backend(monkeypatch, planned = "fp8", verified = False)
monkeypatch.setattr(dmod, "denoiser_prequant_cached", lambda *_a, **_k: False)
backend._run_load(
repo_id = Z_IMAGE_REPO,
model_kind = "pipeline",
transformer_prequant_path = "/models/missing.pt",
_load_token = 1,
)
assert seen["_pipeline_prequant_planned"] is None
def test_the_decline_is_pinned_across_plan_and_load(monkeypatch, hub):
"""A decline is pinned from plan to load, so the loader never re-takes it and fetches inline."""
backend, seen, _fetched = _run_load_backend(monkeypatch, planned = PIPELINE_SEED_DECLINED)
backend._run_load(repo_id = Z_IMAGE_REPO, model_kind = "pipeline", _load_token = 1)
assert seen["_pipeline_prequant_planned"] == PIPELINE_SEED_DECLINED
assert seen["_pipeline_prequant_skipped"] == ()
def test_an_offline_load_never_probes_the_hub_for_an_artifact(monkeypatch):
"""An offline load answers from the cache alone, so it never probes the Hub."""
backend, seen, fetched = _run_load_backend(monkeypatch, planned = "fp8")
def _never(*_a, **_k):
raise AssertionError("the offline path asked the Hub about a checkpoint")
monkeypatch.setattr(DiffusionBackend, "_dit_prequant_plan_source", _never)
monkeypatch.setattr(dmod, "denoiser_prequant_cached", lambda *_a, **_k: False)
backend._run_load(
repo_id = Z_IMAGE_REPO, model_kind = "pipeline", local_files_only = True, _load_token = 1
)
assert seen["_pipeline_prequant_planned"] is None
assert fetched == []
def test_an_offline_reload_seeds_from_the_cached_artifact(monkeypatch):
"""The cache the first load built is reusable: an API reload seeds instead of assembling bf16
from a snapshot whose released denoiser shards that load deliberately left out."""
backend, seen, fetched = _run_load_backend(monkeypatch, planned = "fp8")
def _never(*_a, **_k):
raise AssertionError("the offline path asked the Hub about a checkpoint")
monkeypatch.setattr(DiffusionBackend, "_dit_prequant_plan_source", _never)
monkeypatch.setattr(dmod, "denoiser_prequant_cached", lambda *_a, **_k: True)
backend._run_load(
repo_id = Z_IMAGE_REPO, model_kind = "pipeline", local_files_only = True, _load_token = 1
)
assert seen["_pipeline_prequant_planned"] == "fp8"
assert seen["_pipeline_prequant_skipped"] == ()
assert fetched == []
class _FakePipe:
def __init__(self) -> None:
self.transformer = object()
def to(self, *_a, **_k):
return self
class _FakePipeline:
last: dict = {}
@classmethod
def from_pretrained(cls, base, **kwargs):
_FakePipeline.last = {"base": base, **kwargs}
return _FakePipe()
class _FakeTransformer:
pass
@pytest.fixture
def fake_runtime(monkeypatch):
torch = types.ModuleType("torch")
torch.bfloat16 = "bfloat16"
torch.float16 = "float16"
torch.float32 = "float32"
torch.cuda = types.SimpleNamespace(is_available = lambda: False)
torch.backends = types.SimpleNamespace(mps = None)
torch.inference_mode = lambda: contextlib.nullcontext()
diffusers = types.ModuleType("diffusers")
diffusers.ZImagePipeline = _FakePipeline
diffusers.ZImageTransformer2DModel = _FakeTransformer
diffusers.GGUFQuantizationConfig = lambda compute_dtype = None: ("quant", compute_dtype)
monkeypatch.setitem(sys.modules, "torch", torch)
monkeypatch.setitem(sys.modules, "diffusers", diffusers)
monkeypatch.setattr(dmod, "clear_gpu_cache", lambda: None)
_FakePipeline.last = {}
yield
def _load_backend(
monkeypatch,
*,
seeded = True,
offload = "none",
):
"""A backend whose pipeline assembly, memory plan and quantiser are all observable."""
backend = DiffusionBackend()
monkeypatch.setattr(backend, "_target_for_ordinal", lambda *_a, **_k: _target())
monkeypatch.setattr(dmod, "apply_diffusion_device_ordinal", lambda _t: None)
monkeypatch.setattr(dmod, "prefer_ungated_mirror", lambda base, *_a, **_k: base)
monkeypatch.setattr(dmod, "apply_memory_plan", lambda *_a, **_k: ("none", False))
monkeypatch.setattr(dmod, "raise_on_unified_memory_shortfall", lambda *_a, **_k: None)
monkeypatch.setattr(DiffusionBackend, "_resident_sized_plan", lambda _s, plan, *_a, **_k: plan)
monkeypatch.setattr(dmod, "dense_transformer_supported", lambda _t: True)
monkeypatch.setattr(tqmod, "dense_transformer_supported", lambda _t: True)
monkeypatch.setattr(
dmod, "select_transformer_quant_scheme", lambda target, mode, family = None, **_k: "fp8"
)
monkeypatch.setattr(dmod, "dense_quant_blocker", lambda _pipe, **_kw: None)
monkeypatch.setattr(dmod, "_pipeline_quant_uncompilable_reason", lambda *_a, **_k: None)
monkeypatch.setattr(dmod, "stored_denoiser_precision", lambda *_a, **_k: None)
monkeypatch.setattr(dmod, "denoiser_modules", lambda pipe: [("transformer", object())])
monkeypatch.setattr(
pqmod, "restricted_prequant_load_supported", lambda _scheme, filename = None: True
)
plans: list = []
def _plan_memory(_self, *_a, **kwargs):
plans.append(kwargs)
policy = offload if kwargs.get("transformer_resident_override_mib") is not None else "none"
return types.SimpleNamespace(
offload_policy = policy, requested_mode = None, estimates = {}, reasons = ()
)
monkeypatch.setattr(DiffusionBackend, "_plan_memory", _plan_memory)
quantised: list = []
def _quantize(pipe, target, **kwargs):
quantised.append(kwargs.get("mode"))
return "fp8"
monkeypatch.setattr(dmod, "quantize_transformer", _quantize)
seeds: list = []
def _load_prequantized(_cls, base, source, **kwargs):
seeds.append({"base": base, "source": source, **kwargs})
return _FakeTransformer() if seeded else None
monkeypatch.setattr(pqmod, "load_prequantized_transformer", _load_prequantized)
restored: list = []
monkeypatch.setattr(
DiffusionBackend,
"_prefetch_files",
lambda _self, repo_id, gguf, base, files, *_a, **_k: restored.append(tuple(files)),
)
return backend, types.SimpleNamespace(
plans = plans, quantised = quantised, seeds = seeds, restored = restored
)
def _load(backend, **overrides):
kwargs = {
"model_kind": "pipeline",
"_pipeline_prequant_planned": "fp8",
"_pipeline_prequant_skipped": tuple(DENOISER_SHARDS),
**overrides,
}
return backend.load_pipeline(Z_IMAGE_REPO, **kwargs)
def test_a_seeded_denoiser_engages_the_quant_without_a_second_conversion(fake_runtime, monkeypatch):
backend, spy = _load_backend(monkeypatch)
status = _load(backend)
assert isinstance(_FakePipeline.last["transformer"], _FakeTransformer)
assert status["transformer_quant"] == "fp8"
assert spy.quantised == []
assert spy.restored == []
seed = spy.seeds[0]
assert seed["source"].location == PREQUANT_REPO
assert seed["source"].filename == PREQUANT_FILE
assert seed["scheme"] == "fp8"
assert seed["min_features"] == dmod.DEFAULT_MIN_LINEAR_FEATURES
assert seed["cache_dir"] == dmod.hub_cache_dir()
def test_the_resolved_record_names_the_hosted_file(fake_runtime, monkeypatch):
"""``resolved`` names the hosted FILE, which renders differently from a runtime quantise."""
backend, _spy = _load_backend(monkeypatch)
resolved = _load(backend)["resolved"]["transformer_quant"]
assert resolved["value"] == "fp8"
assert resolved["artifact"] == f"prequant:{PREQUANT_REPO}/{PREQUANT_FILE}"
assert PREQUANT_FILE in resolved["reason"]
assert resolved["source"] == "auto"
def test_a_seed_that_does_not_land_replans_and_tops_up_the_shards(fake_runtime, monkeypatch):
"""A failed seed re-plans at bf16 and restores shards from_pretrained cannot re-fetch."""
backend, spy = _load_backend(monkeypatch, seeded = False)
status = _load(backend)
assert "transformer" not in _FakePipeline.last
assert spy.restored and set(DENOISER_SHARDS) <= set(spy.restored[0])
assert spy.quantised == ["auto"]
assert status["transformer_quant"] == "fp8"
assert status["resolved"]["transformer_quant"].get("artifact") is None
def test_a_top_up_that_spans_cache_roots_assembles_from_the_hub_id(fake_runtime, monkeypatch):
"""Split cache roots: no snapshot holds both, so assemble from the hub id, not the staged directory."""
backend, _spy = _load_backend(monkeypatch, seeded = False)
monkeypatch.setattr(DiffusionBackend, "_prefetch_files", lambda *_a, **_k: None)
_load(backend, _base_local_dir = "/old/root/snapshots/abc")
assert _FakePipeline.last["base"] == Z_IMAGE_REPO
def test_a_top_up_never_promotes_a_snapshot_the_staging_did_not_hand_back(
fake_runtime, monkeypatch
):
"""The top-up stages the shards only. A snapshot the staging never returned holds them without
the companions, so it must not become the directory the pipeline assembles from."""
backend, _spy = _load_backend(monkeypatch, seeded = False)
monkeypatch.setattr(
DiffusionBackend, "_prefetch_files", lambda *_a, **_k: "/live/root/snapshots/abc"
)
_load(backend, _base_local_dir = None)
assert _FakePipeline.last["base"] == Z_IMAGE_REPO
def test_a_single_root_top_up_keeps_assembling_from_the_staged_snapshot(fake_runtime, monkeypatch):
"""Shards land in the staged snapshot, which keeps from_pretrained off the hub."""
backend, _spy = _load_backend(monkeypatch, seeded = False)
monkeypatch.setattr(
DiffusionBackend, "_prefetch_files", lambda *_a, **_k: "/live/root/snapshots/abc"
)
_load(backend, _base_local_dir = "/live/root/snapshots/abc")
assert _FakePipeline.last["base"] == "/live/root/snapshots/abc"
def test_an_artifact_sized_plan_that_offloads_at_load_time_drops_the_seed(
fake_runtime, monkeypatch
):
"""A plan that offloads at load time drops the seed: torchao tensors reject offload hooks."""
backend, spy = _load_backend(monkeypatch, offload = "sequential")
_load(backend)
assert spy.seeds == []
assert spy.restored and set(DENOISER_SHARDS) <= set(spy.restored[0])
def test_the_artifact_label_names_the_file_that_really_loaded():
"""A repo serving only its fallback filename is labelled with the fallback, not the primary."""
# _resolve_checkpoint_path may fall back, so source.filename may not be the fetched file
import types
from core.inference.diffusion_denoiser_prequant import prequant_artifact_label
source = types.SimpleNamespace(
kind = "repo", location = "unsloth/Z-Image-Turbo-FP8", filename = "Z-Image-Turbo-FP8.pt"
)
assert prequant_artifact_label(source) == (
"prequant:unsloth/Z-Image-Turbo-FP8/Z-Image-Turbo-FP8.pt"
)
loaded = types.SimpleNamespace(_unsloth_prequant_path = "/cache/blobs/Z-Image-Turbo-fp8.pt")
assert prequant_artifact_label(source, loaded) == (
"prequant:unsloth/Z-Image-Turbo-FP8/Z-Image-Turbo-fp8.pt"
)
local = types.SimpleNamespace(kind = "path", location = "/models/mine.pt", filename = None)
assert prequant_artifact_label(local, loaded) == "prequant:/models/mine.pt"
def test_a_dropped_seed_replans_once_the_dense_shards_are_back(fake_runtime, monkeypatch):
"""The plan the dense load runs on is taken AFTER the skipped transformer shards are restored."""
# a plan made while transformer/ was skipped prices companions only and keeps bf16 resident
backend, spy = _load_backend(monkeypatch, offload = "sequential")
_load(backend)
assert spy.seeds == []
assert spy.restored and set(DENOISER_SHARDS) <= set(spy.restored[0])
assert len(spy.plans) == 3
assert spy.plans[-1].get("transformer_resident_override_mib") is None
def test_nothing_skipped_takes_no_extra_plan(fake_runtime, monkeypatch):
"""A load that skipped no shards re-plans nothing: there is no under-counted plan to redo."""
backend, spy = _load_backend(monkeypatch, offload = "sequential")
_load(backend, _pipeline_prequant_skipped = ())
assert spy.restored == []
assert len(spy.plans) == 2
def test_a_declined_plan_never_seeds_at_the_load(fake_runtime, monkeypatch):
backend, spy = _load_backend(monkeypatch)
_load(
backend,
_pipeline_prequant_planned = PIPELINE_SEED_DECLINED,
_pipeline_prequant_skipped = (),
)
assert spy.seeds == []
assert spy.restored == []
assert spy.quantised == ["auto"]
def test_a_direct_load_with_no_plan_phase_keeps_todays_behaviour(fake_runtime, monkeypatch):
"""A direct call with no plan phase never seeds a decision the pull was not scoped on."""
backend, spy = _load_backend(monkeypatch)
_load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())
assert spy.seeds == []
assert spy.quantised == ["auto"]
def test_an_explicit_scheme_with_a_hosted_artifact_is_seeded_too(monkeypatch):
backend = _settle_backend(monkeypatch)
assert _settle(backend, transformer_quant = "fp8") == "fp8"
def test_an_explicit_scheme_with_no_artifact_falls_to_the_in_memory_path(fake_runtime, monkeypatch):
"""With nothing to seed, an explicit scheme quantises in place and still fails closed."""
backend, spy = _load_backend(monkeypatch)
status = _load(
backend,
transformer_quant = "fp8",
_pipeline_prequant_planned = None,
_pipeline_prequant_skipped = (),
)
assert spy.seeds == []
assert spy.quantised == ["fp8"]
assert status["transformer_quant"] == "fp8"
monkeypatch.setattr(dmod, "quantize_transformer", lambda *_a, **_k: None)
monkeypatch.setattr(dmod, "transformer_is_quantised", lambda _m: False)
with pytest.raises(RuntimeError, match = "transformer_quant='fp8'"):
_load(
backend,
transformer_quant = "fp8",
_pipeline_prequant_planned = None,
_pipeline_prequant_skipped = (),
)
@pytest.mark.parametrize(
"family, repo",
[
("z-image", "unsloth/Z-Image-Turbo-FP8"),
("flux.1-schnell", "unsloth/FLUX.1-schnell-FP8"),
("qwen-image", "unsloth/Qwen-Image-FP8"),
],
)
def test_the_rebuilt_fp8_artifacts_are_listed_for_both_schemes(family, repo):
"""The hosted repos carry an fp8 and an int8 denoiser file, so an fp8 host seeds instead of quantising in memory."""
from core.inference.diffusion_families import detect_family, family_prequant_repo
fam = detect_family(family)
assert fam is not None
for scheme in ("fp8", "int8"):
assert family_prequant_repo(fam, scheme) == repo
def _encoders_streamed(**_k):
return types.SimpleNamespace(offload_policy = "group", stream_transformer = False)
def test_an_artifact_that_fits_once_the_encoders_stream_is_seeded(monkeypatch):
"""An artifact-sized plan streaming only the encoders keeps the torchao seed."""
backend = _settle_backend(monkeypatch)
monkeypatch.setattr(DiffusionBackend, "_plan_memory", lambda *_a, **k: _encoders_streamed(**k))
assert _settle(backend) == "fp8"
def test_an_artifact_plan_that_streams_the_transformer_still_declines(monkeypatch):
backend = _settle_backend(monkeypatch)
monkeypatch.setattr(
DiffusionBackend,
"_plan_memory",
lambda *_a, **_k: types.SimpleNamespace(offload_policy = "group", stream_transformer = True),
)
assert _settle(backend) == PIPELINE_SEED_DECLINED
@pytest.mark.parametrize("policy", ["group", "streaming"])
def test_an_artifact_plan_that_streams_the_transformer_seeds_on_a_measured_torchao(
monkeypatch, policy
):
from core.inference import diffusion_memory
monkeypatch.setattr(diffusion_memory, "_installed_torchao_version", lambda: (0, 18))
monkeypatch.setattr(diffusion_memory, "_torchao_stream_pinnable", lambda plan, *_a: True)
backend = _settle_backend(monkeypatch)
monkeypatch.setattr(
DiffusionBackend,
"_plan_memory",
lambda *_a, **_k: types.SimpleNamespace(offload_policy = policy, stream_transformer = True),
)
assert _settle(backend) == "fp8"
def test_a_resident_rung_still_beats_a_streamed_one(monkeypatch):
from core.inference import diffusion_memory
monkeypatch.setattr(diffusion_memory, "_installed_torchao_version", lambda: (0, 18))
monkeypatch.setattr(diffusion_memory, "_torchao_stream_pinnable", lambda plan, *_a: True)
backend = _settle_backend_walking(
monkeypatch, artifacts = ("int8", "fp8"), candidates = ("int8", "fp8")
)
monkeypatch.setattr(
DiffusionBackend,
"_plan_memory",
lambda *_a, **k: types.SimpleNamespace(
offload_policy = "group" if k["transformer_resident_override_mib"] >= 31_000 else "none",
stream_transformer = True,
),
)
assert _settle(backend) == "fp8"
backend = _settle_backend_walking(monkeypatch, artifacts = ("int8",), candidates = ("int8", "fp8"))
monkeypatch.setattr(
DiffusionBackend,
"_plan_memory",
lambda *_a, **_k: types.SimpleNamespace(offload_policy = "group", stream_transformer = True),
)
assert _settle(backend) == "int8"
def test_a_seed_whose_load_plan_streams_only_the_encoders_is_kept(fake_runtime, monkeypatch):
backend, spy = _load_backend(monkeypatch, offload = "group")
real_plan = DiffusionBackend._plan_memory
def _plan(self, *a, **kwargs):
plan = real_plan(self, *a, **kwargs)
if kwargs.get("transformer_resident_override_mib") is not None:
plan.stream_transformer = False
return plan
monkeypatch.setattr(DiffusionBackend, "_plan_memory", _plan)
status = _load(backend)
assert spy.seeds and spy.restored == []
assert status["transformer_quant"] == "fp8"
_MEASURED = {"safe_device_budget_mib": 170_000, "resident_required_mib": 60_000}
def _uncompilable(monkeypatch, *, measured = True):
monkeypatch.setattr(dmod, "family_compiles_regionally", lambda _fam: False)
monkeypatch.setattr(dmod, "family_bf16_components_gb", lambda *_a, **_k: (90.0, 10.0, 0.2))
inner = DiffusionBackend._plan_memory
def _plan(self, *a, **k):
plan = inner(self, *a, **k)
plan.estimates = dict(_MEASURED) if measured else {}
return plan
monkeypatch.setattr(DiffusionBackend, "_plan_memory", _plan)
def _offload_when_bf16_sized(monkeypatch, bf16_policy):
monkeypatch.setattr(
DiffusionBackend,
"_plan_memory",
lambda *_a, **k: types.SimpleNamespace(
offload_policy = bf16_policy
if (k.get("transformer_resident_override_mib") or 0) > 80_000
else "none"
),
)
def test_an_uncompilable_family_keeps_the_released_weights_that_fit(monkeypatch):
backend = _settle_backend(monkeypatch)
_offload_when_bf16_sized(monkeypatch, "none")
_uncompilable(monkeypatch)
assert _settle(backend) is None
def test_an_uncompilable_family_still_seeds_when_the_budget_is_unmeasured(monkeypatch):
backend = _settle_backend(monkeypatch)
_offload_when_bf16_sized(monkeypatch, "none")
_uncompilable(monkeypatch, measured = False)
assert _settle(backend) == "fp8"
def test_an_uncompilable_family_still_seeds_when_the_released_weights_would_offload(monkeypatch):
backend = _settle_backend(monkeypatch)
_offload_when_bf16_sized(monkeypatch, "sequential")
_uncompilable(monkeypatch)
assert _settle(backend) == "fp8"
def test_an_uncompilable_family_still_seeds_an_explicit_scheme(monkeypatch):
backend = _settle_backend(monkeypatch)
_offload_when_bf16_sized(monkeypatch, "none")
_uncompilable(monkeypatch)
assert _settle(backend, transformer_quant = "fp8") == "fp8"
def _operator_checkpoint(monkeypatch, tmp_path):
ckpt = tmp_path / "transformer_fp8.pt"
ckpt.write_bytes(b"x")
monkeypatch.setenv(pqmod.ALLOW_LOCAL_PREQUANT_PATH_ENV, str(tmp_path))
return str(ckpt)
def test_an_uncompilable_family_still_plans_an_operators_checkpoint(monkeypatch, tmp_path):
backend = _settle_backend(monkeypatch)
_offload_when_bf16_sized(monkeypatch, "none")
_uncompilable(monkeypatch)
path = _operator_checkpoint(monkeypatch, tmp_path)
monkeypatch.setattr(pqmod, "local_prequant_scheme", lambda _path: "fp8")
assert _settle(backend, transformer_prequant_path = path) is not None
assert _settle(backend, transformer_prequant_path = str(tmp_path / "missing.pt")) is None
def test_a_compilable_family_seeds_as_before(monkeypatch):
backend = _settle_backend(monkeypatch)
monkeypatch.setattr(dmod, "family_compiles_regionally", lambda _fam: True)
_offload_when_bf16_sized(monkeypatch, "none")
assert _settle(backend) == "fp8"
def test_an_uncompilable_family_loads_auto_unquantised(fake_runtime, monkeypatch):
backend, spy = _load_backend(monkeypatch)
_uncompilable(monkeypatch)
status = _load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())
assert spy.quantised == []
assert spy.seeds == []
resolved = status["resolved"]["transformer_quant"]
assert (resolved["value"], resolved["source"], resolved["status"]) == ("off", "auto", "applied")
assert "cannot be regionally compiled" in resolved["reason"]
def test_an_uncompilable_family_loads_an_operators_checkpoint_under_auto(
fake_runtime, monkeypatch, tmp_path
):
backend, spy = _load_backend(monkeypatch)
_uncompilable(monkeypatch)
status = _load(
backend,
transformer_prequant_path = _operator_checkpoint(monkeypatch, tmp_path),
_pipeline_prequant_planned = None,
_pipeline_prequant_skipped = (),
)
assert spy.quantised == ["auto"]
assert "cannot be regionally compiled" not in str(status["resolved"]["transformer_quant"])
def test_a_compilable_family_loads_without_the_bf16_replan(fake_runtime, monkeypatch):
backend, spy = _load_backend(monkeypatch)
monkeypatch.setattr(dmod, "family_compiles_regionally", lambda _fam: True)
monkeypatch.setattr(dmod, "auto_bf16_when_resident_reason", lambda _name: None)
replans = []
monkeypatch.setattr(
DiffusionBackend,
"_bf16_resident_plan",
lambda self, plan, *a, **k: replans.append(1) or plan,
)
_load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())
assert replans == []
assert spy.quantised == ["auto"]
def test_an_uncompilable_family_quantises_auto_when_bf16_would_offload(fake_runtime, monkeypatch):
backend, spy = _load_backend(monkeypatch)
_uncompilable(monkeypatch)
real_plan = DiffusionBackend._plan_memory
def _plan(self, *a, **k):
plan = real_plan(self, *a, **k)
if k.get("transformer_resident_override_mib") is None:
plan.offload_policy = "sequential"
return plan
monkeypatch.setattr(DiffusionBackend, "_plan_memory", _plan)
_load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())
assert spy.quantised == ["auto"]
def test_an_uncompilable_family_honours_an_explicit_scheme(fake_runtime, monkeypatch):
backend, spy = _load_backend(monkeypatch)
_uncompilable(monkeypatch)
_load(
backend,
transformer_quant = "fp8",
_pipeline_prequant_planned = None,
_pipeline_prequant_skipped = (),
)
assert spy.quantised == ["fp8"]
def test_an_uncompilable_family_on_a_host_without_dense_quant_reports_as_before(
fake_runtime, monkeypatch
):
backend, spy = _load_backend(monkeypatch)
_uncompilable(monkeypatch)
monkeypatch.setattr(dmod, "dense_transformer_supported", lambda _t: False)
status = _load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())
assert spy.quantised == []
assert "regionally compiled" not in status["resolved"]["transformer_quant"]["reason"]
def test_an_uncompilable_family_quantises_auto_when_the_budget_is_unmeasured(
fake_runtime, monkeypatch
):
backend, spy = _load_backend(monkeypatch)
_uncompilable(monkeypatch, measured = False)
_load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())
assert spy.quantised == ["auto"]
def _measured_bf16_family(monkeypatch, *, measured = True):
_uncompilable(monkeypatch, measured = measured)
monkeypatch.setattr(dmod, "family_compiles_regionally", lambda _fam: True)
monkeypatch.setattr(dmod, "auto_bf16_when_resident_reason", lambda _name: "measured: no faster")
def test_a_measured_bf16_family_keeps_the_released_weights_that_fit(monkeypatch):
backend = _settle_backend(monkeypatch)
_offload_when_bf16_sized(monkeypatch, "none")
_measured_bf16_family(monkeypatch)
assert _settle(backend) is None
def test_a_measured_bf16_family_seeds_when_the_released_weights_would_offload(monkeypatch):
backend = _settle_backend(monkeypatch)
_offload_when_bf16_sized(monkeypatch, "sequential")
_measured_bf16_family(monkeypatch)
assert _settle(backend) == "fp8"
def test_a_measured_bf16_family_loads_auto_unquantised_with_the_measured_reason(
fake_runtime, monkeypatch
):
backend, spy = _load_backend(monkeypatch)
_measured_bf16_family(monkeypatch)
status = _load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())
assert spy.quantised == [] and spy.seeds == []
resolved = status["resolved"]["transformer_quant"]
assert (resolved["value"], resolved["source"]) == ("off", "auto")
assert "measured: no faster" in resolved["reason"]
assert "cannot be regionally compiled" not in resolved["reason"]
def test_a_measured_bf16_family_quantises_auto_when_the_budget_is_unmeasured(
fake_runtime, monkeypatch
):
backend, spy = _load_backend(monkeypatch)
_measured_bf16_family(monkeypatch, measured = False)
_load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())
assert spy.quantised == ["auto"]
@pytest.mark.parametrize(
"repo, kept",
[
("Alpha-VLLM/Lumina-Image-2.0", True),
("HiDream-ai/HiDream-I1-Full", True),
("black-forest-labs/FLUX.1-dev", False),
("Tongyi-MAI/Z-Image-Turbo", True),
("black-forest-labs/FLUX.2-klein-4B", True),
("black-forest-labs/FLUX.2-klein-base-4B", True),
("black-forest-labs/flux-3-image-klein", True),
("black-forest-labs/flux-3-image-klein-base", True),
("black-forest-labs/FLUX.2-dev", False),
("Qwen/Qwen-Image", False),
# A 5B DiT: INT8-ConvRot doubles LPIPS against the eager port, so bf16 while it fits.
("OpenMOSS-Team/MOSS-Image-SFT", True),
("OpenMOSS-Team/MOSS-Image-Base", True),
],
)
def test_the_bf16_rule_names_only_the_measured_families(repo, kept):
fam = detect_family_for_pick(repo, None, None)
assert fam is not None
reason = dmod._auto_keeps_bf16_reason(fam)
assert (reason is not None) is kept
assert reason is None or "cannot be regionally compiled" not in reason
assert dmod._auto_keeps_bf16_reason(fam, "gguf") is None
def test_an_uncompilable_family_keeps_its_reason_for_a_gguf_load(monkeypatch):
fam = detect_family_for_pick("Alpha-VLLM/Lumina-Image-2.0", None, None)
monkeypatch.setattr(dmod, "family_compiles_regionally", lambda _fam: False)
assert "cannot be regionally compiled" in dmod._auto_keeps_bf16_reason(fam, "gguf")
def test_a_resident_plan_whose_requirement_exceeds_the_budget_proves_no_fit():
fits = types.SimpleNamespace(
offload_policy = "none",
estimates = {"safe_device_budget_mib": 60_000, "resident_required_mib": 60_000},
)
over = types.SimpleNamespace(
offload_policy = "none",
estimates = {"safe_device_budget_mib": 50_000, "resident_required_mib": 60_000},
)
assert dmod._plan_proves_resident(fits)
assert not dmod._plan_proves_resident(over)
def test_an_uncompilable_family_seeds_when_bf16_overflows_a_unified_pool(monkeypatch):
backend = _settle_backend(monkeypatch)
monkeypatch.setattr(dmod, "family_compiles_regionally", lambda _fam: False)
monkeypatch.setattr(dmod, "family_bf16_components_gb", lambda *_a, **_k: (90.0, 10.0, 0.2))
monkeypatch.setattr(
DiffusionBackend,
"_plan_memory",
lambda *_a, **_k: types.SimpleNamespace(
offload_policy = "none",
estimates = {"safe_device_budget_mib": 50_000, "resident_required_mib": 60_000},
),
)
assert _settle(backend) == "fp8"
def test_an_uncompilable_family_prices_a_pre_cast_text_encoder(monkeypatch):
import core.inference.diffusion_te_prequant as teq
backend = _settle_backend(monkeypatch)
monkeypatch.setattr(
teq,
"te_prequant_budget_scale",
lambda _fam, *, te_quant_mode, target, base: 0.5 if te_quant_mode == "fp8" else 1.0,
)
monkeypatch.setattr(
DiffusionBackend,
"_plan_memory",
lambda *_a, **k: types.SimpleNamespace(
offload_policy = "none"
if (k.get("companion_override_mib") or 0) < 6_000
or (k.get("transformer_resident_override_mib") or 0) < 80_000
else "sequential"
),
)
_uncompilable(monkeypatch)
assert _settle(backend) == "fp8"
assert _settle(backend, text_encoder_quant = "fp8") is None
def test_an_offline_uncompilable_pick_keeps_a_cached_seed_without_released_shards(monkeypatch):
backend = _settle_backend(monkeypatch)
_offload_when_bf16_sized(monkeypatch, "none")
_uncompilable(monkeypatch)
monkeypatch.setattr(
DiffusionBackend, "_released_transformer_cached", staticmethod(lambda _base: False)
)
assert _settle(backend, local_files_only = True) == "fp8"
assert _settle(backend) is None
monkeypatch.setattr(
DiffusionBackend, "_released_transformer_cached", staticmethod(lambda _base: True)
)
assert _settle(backend, local_files_only = True) is None
def test_released_transformer_cached_reads_the_transformer_folder(tmp_path):
(tmp_path / "vae").mkdir()
(tmp_path / "vae" / "diffusion_pytorch_model.safetensors").write_bytes(b"x")
assert not DiffusionBackend._released_transformer_cached(str(tmp_path))
(tmp_path / "transformer").mkdir()
(tmp_path / "transformer" / "diffusion_pytorch_model.safetensors").write_bytes(b"x")
assert DiffusionBackend._released_transformer_cached(str(tmp_path))
assert not DiffusionBackend._released_transformer_cached(None)
def test_the_seed_planner_hears_the_encoder_choice_and_offline_flag(monkeypatch):
backend, _seen, _fetched = _run_load_backend(monkeypatch, planned = None)
heard: dict = {}
monkeypatch.setattr(
DiffusionBackend,
"_pipeline_planned_denoiser_scheme",
lambda _self, _fam, **k: heard.update(k),
)
backend._run_load(
repo_id = Z_IMAGE_REPO, model_kind = "pipeline", local_files_only = True, _load_token = 1
)
assert heard["local_files_only"] is True
assert "text_encoder_quant" in heard
def test_an_uncompilable_family_keeps_bf16_when_the_resident_table_fits(fake_runtime, monkeypatch):
backend, spy = _load_backend(monkeypatch)
_uncompilable(monkeypatch)
real_plan = DiffusionBackend._plan_memory
overrides: list = []
def _plan(self, *a, **k):
plan = real_plan(self, *a, **k)
overrides.append(k.get("transformer_resident_override_mib"))
if k.get("transformer_resident_override_mib") is None:
plan.offload_policy = "sequential"
return plan
monkeypatch.setattr(DiffusionBackend, "_plan_memory", _plan)
monkeypatch.setattr(
DiffusionBackend,
"_resident_sized_plan",
lambda _s, plan, *_a, **_k: types.SimpleNamespace(**vars(plan)),
)
status = _load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())
assert spy.quantised == []
assert int(90.0 * 1000.0**3 / (1024.0 * 1024.0)) in overrides
assert status["resolved"]["transformer_quant"]["value"] == "off"
def _write_index(folder, shards):
folder.mkdir(parents = True, exist_ok = True)
(folder / "diffusion_pytorch_model.safetensors.index.json").write_text(
json.dumps({"weight_map": {f"w{i}": shard for i, shard in enumerate(shards)}})
)
def test_a_partial_sharded_transformer_is_not_a_cached_release(tmp_path):
shards = [f"diffusion_pytorch_model-0000{i}-of-00002.safetensors" for i in (1, 2)]
_write_index(tmp_path / "transformer", shards)
(tmp_path / "transformer" / shards[0]).write_bytes(b"x")
assert not DiffusionBackend._released_transformer_cached(str(tmp_path))
(tmp_path / "transformer" / shards[1]).write_bytes(b"x")
assert DiffusionBackend._released_transformer_cached(str(tmp_path))
def test_a_complete_bin_set_does_not_cover_a_partial_safetensors_index(tmp_path):
# diffusers loads the safetensors index, never a .bin index by default
folder = tmp_path / "transformer"
bins = [f"diffusion_pytorch_model-0000{i}-of-00002.bin" for i in (1, 2)]
folder.mkdir()
(folder / "diffusion_pytorch_model.bin.index.json").write_text(
json.dumps({"weight_map": {"a": bins[0], "b": bins[1]}})
)
for shard in bins:
(folder / shard).write_bytes(b"x")
assert not DiffusionBackend._released_transformer_cached(str(tmp_path))
shards = [f"diffusion_pytorch_model-0000{i}-of-00002.safetensors" for i in (1, 2)]
_write_index(folder, shards)
(folder / shards[0]).write_bytes(b"x")
assert not DiffusionBackend._released_transformer_cached(str(tmp_path))
(folder / shards[1]).write_bytes(b"x")
assert DiffusionBackend._released_transformer_cached(str(tmp_path))
def test_shards_scattered_across_revisions_are_not_a_cached_release(monkeypatch, tmp_path):
repo_dir = tmp_path / "models--org--repo"
shards = [f"diffusion_pytorch_model-0000{i}-of-00002.safetensors" for i in (1, 2)]
for rev, shard in (("aaa", shards[0]), ("bbb", shards[1])):
folder = repo_dir / "snapshots" / rev / "transformer"
_write_index(folder, shards)
(folder / shard).write_bytes(b"x")
(repo_dir / "snapshots" / "aaa" / "model_index.json").write_text("{}")
(repo_dir / "refs").mkdir()
(repo_dir / "refs" / "main").write_text("aaa")
monkeypatch.setattr(
DiffusionBackend, "_hub_cache_repo_dirs", staticmethod(lambda _repo: [repo_dir])
)
assert not DiffusionBackend._released_transformer_cached("org/repo")
(repo_dir / "snapshots" / "aaa" / "transformer" / shards[1]).write_bytes(b"x")
assert DiffusionBackend._released_transformer_cached("org/repo")
def test_hidreams_pre_cast_fourth_encoder_is_priced_in_the_bf16_plan(monkeypatch):
import core.inference.diffusion_te_prequant as teq
from core.inference.diffusion_hidream import HIDREAM_LLAMA_BF16_BYTES
fam = detect_family_for_pick("HiDream-ai/HiDream-I1-Full", None, None)
assert fam is not None and fam.name == "hidream-i1"
monkeypatch.setattr(teq, "te_prequant_budget_scale", lambda *_a, **_k: 1.0)
monkeypatch.setattr(
teq,
"te_prequant_sources_for_base",
lambda _fam, _base, *, te_quant_mode, target, components = (), **_k: (
{"text_encoder_4": object()}
if te_quant_mode == "fp8" and tuple(components) == ("text_encoder_4",)
else {}
),
)
seen: list = []
monkeypatch.setattr(DiffusionBackend, "_plan_memory", lambda _s, *_a, **k: seen.append(k) or k)
backend = DiffusionBackend()
kwargs = {"kind": "pipeline", "repo_id": "HiDream-ai/HiDream-I1-Full"}
backend._bf16_table_plan(_target(), fam, "HiDream-ai/HiDream-I1-Full", None, False, **kwargs)
backend._bf16_table_plan(
_target(),
fam,
"HiDream-ai/HiDream-I1-Full",
None,
False,
text_encoder_quant = "fp8",
**kwargs,
)
dense, pre_cast = seen[0]["text_encoder_override_mib"], seen[1]["text_encoder_override_mib"]
saved = HIDREAM_LLAMA_BF16_BYTES * (1.0 - teq.TE_PREQUANT_BUDGET_SCALE) / (1024 * 1024)
assert abs((dense - pre_cast) - saved) <= 2
assert seen[1]["companion_override_mib"] < seen[0]["companion_override_mib"]
def test_a_lone_numbered_shard_without_its_index_is_not_a_cached_release(tmp_path):
folder = tmp_path / "transformer"
folder.mkdir()
(folder / "diffusion_pytorch_model-00001-of-00002.safetensors").write_bytes(b"x")
assert not DiffusionBackend._released_transformer_cached(str(tmp_path))
(folder / "diffusion_pytorch_model.safetensors").write_bytes(b"x")
assert DiffusionBackend._released_transformer_cached(str(tmp_path))
def test_an_offline_pick_budgets_the_encoder_dense(monkeypatch):
import core.inference.diffusion_te_prequant as teq
backend = _settle_backend(monkeypatch)
monkeypatch.setattr(
teq,
"te_prequant_budget_scale",
lambda _fam, *, te_quant_mode, target, base: 0.5 if te_quant_mode == "fp8" else 1.0,
)
monkeypatch.setattr(
DiffusionBackend,
"_plan_memory",
lambda *_a, **k: types.SimpleNamespace(
offload_policy = "none"
if (k.get("companion_override_mib") or 0) < 6_000
or (k.get("transformer_resident_override_mib") or 0) < 80_000
else "sequential"
),
)
_uncompilable(monkeypatch)
monkeypatch.setattr(
DiffusionBackend, "_released_transformer_cached", staticmethod(lambda _base: True)
)
assert _settle(backend, text_encoder_quant = "fp8") is None
assert _settle(backend, text_encoder_quant = "fp8", local_files_only = True) == "fp8"
def test_an_offline_pick_finds_released_shards_under_the_mirror(monkeypatch):
backend = _settle_backend(monkeypatch)
_offload_when_bf16_sized(monkeypatch, "none")
_uncompilable(monkeypatch)
monkeypatch.setattr(dmod, "prefer_ungated_mirror", lambda base, *_a, **_k: "unsloth/mirror")
monkeypatch.setattr(
DiffusionBackend,
"_released_transformer_cached",
staticmethod(lambda base: base == "unsloth/mirror"),
)
assert _settle(backend, local_files_only = True) is None
def test_a_hidream_pipeline_is_repriced_with_its_standalone_encoder(monkeypatch):
fam = detect_family_for_pick("HiDream-ai/HiDream-I1-Dev", None, None)
assert fam is not None and fam.name == "hidream-i1"
as_built = types.SimpleNamespace(offload_policy = "none", estimates = {}, device_memory = None)
monkeypatch.setattr(DiffusionBackend, "_resident_sized_plan", lambda _s, plan, *_a, **_k: plan)
monkeypatch.setattr(
DiffusionBackend,
"_plan_memory",
lambda _s, *_a, **k: types.SimpleNamespace(offload_policy = "group", overrides = k),
)
repriced = DiffusionBackend()._bf16_resident_plan(
as_built,
_target(),
fam,
"HiDream-ai/HiDream-I1-Dev",
None,
False,
kind = "pipeline",
repo_id = "HiDream-ai/HiDream-I1-Dev",
)
assert repriced is not as_built
table_gb = 34.2 + 28.8
total_mib = (
repriced.overrides["transformer_resident_override_mib"]
+ repriced.overrides["companion_override_mib"]
)
assert total_mib >= int(table_gb * 1000.0**3 / (1024 * 1024)) - 2
def test_the_offline_shard_check_reads_the_root_the_loader_reads(monkeypatch, tmp_path):
live, other = tmp_path / "live" / "models--org--repo", tmp_path / "other" / "models--org--repo"
for repo_dir, with_transformer in ((live, False), (other, True)):
snapshot = repo_dir / "snapshots" / "rev"
snapshot.mkdir(parents = True)
(snapshot / "model_index.json").write_text("{}")
if with_transformer:
(snapshot / "transformer").mkdir()
(snapshot / "transformer" / "diffusion_pytorch_model.safetensors").write_bytes(b"x")
(repo_dir / "refs").mkdir()
(repo_dir / "refs" / "main").write_text("rev")
monkeypatch.setattr(
DiffusionBackend, "_hub_cache_repo_dirs", staticmethod(lambda _repo: [live, other])
)
assert not DiffusionBackend._released_transformer_cached("org/repo")
(live / "snapshots" / "rev" / "model_index.json").unlink()
assert DiffusionBackend._released_transformer_cached("org/repo")
@pytest.mark.parametrize("resolved", [{}, {"text_encoder": ("repo", [("te.safetensors", 1)])}])
def test_pre_cast_sizing_waits_for_a_resolved_encoder_artifact(monkeypatch, resolved):
backend, _seen, _fetched = _run_load_backend(monkeypatch, planned = None)
monkeypatch.setattr(DiffusionBackend, "_te_prequant_plan_files", lambda *_a, **_k: resolved)
heard: dict = {}
monkeypatch.setattr(
DiffusionBackend,
"_pipeline_planned_denoiser_scheme",
lambda _self, _fam, **k: heard.update(k),
)
backend._run_load(
repo_id = Z_IMAGE_REPO, model_kind = "pipeline", text_encoder_quant = "fp8", _load_token = 1
)
assert heard["text_encoder_quant"] == ("fp8" if resolved else None)
def test_a_dtype_variant_twin_is_not_a_cached_release(tmp_path):
folder = tmp_path / "transformer"
shards = [f"diffusion_pytorch_model-0000{i}-of-00002.fp16.safetensors" for i in (1, 2)]
folder.mkdir()
(folder / "diffusion_pytorch_model.fp16.safetensors.index.json").write_text(
json.dumps({"weight_map": {"a": shards[0], "b": shards[1]}})
)
for shard in shards:
(folder / shard).write_bytes(b"x")
(folder / "diffusion_pytorch_model.fp16.safetensors").write_bytes(b"x")
assert not DiffusionBackend._released_transformer_cached(str(tmp_path))
def test_an_offline_pick_checks_only_the_repo_the_load_reads(monkeypatch):
backend = _settle_backend(monkeypatch)
_offload_when_bf16_sized(monkeypatch, "none")
_uncompilable(monkeypatch)
monkeypatch.setattr(dmod, "prefer_ungated_mirror", lambda base, *_a, **_k: base)
monkeypatch.setattr(
DiffusionBackend,
"_released_transformer_cached",
staticmethod(lambda base: base == "unsloth/mirror"),
)
assert _settle(backend, local_files_only = True) == "fp8"
def test_an_auxiliary_checkpoint_is_not_a_cached_release(tmp_path):
folder = tmp_path / "transformer"
folder.mkdir()
(folder / "adapter_model.safetensors.index.json").write_text(
json.dumps({"weight_map": {"a": "adapter_model-00001-of-00001.safetensors"}})
)
(folder / "adapter_model-00001-of-00001.safetensors").write_bytes(b"x")
(folder / "adapter_model.safetensors").write_bytes(b"x")
assert not DiffusionBackend._released_transformer_cached(str(tmp_path))
@pytest.mark.parametrize("resolved", [{}, {"text_encoder": ("repo", [("te.safetensors", 1)])}])
def test_the_load_hears_whether_the_pre_cast_encoder_resolved(monkeypatch, resolved):
backend, seen, _fetched = _run_load_backend(monkeypatch, planned = None)
monkeypatch.setattr(DiffusionBackend, "_te_prequant_plan_files", lambda *_a, **_k: resolved)
backend._run_load(
repo_id = Z_IMAGE_REPO, model_kind = "pipeline", text_encoder_quant = "fp8", _load_token = 1
)
assert seen["_te_prequant_resolved"] is bool(resolved)
@pytest.mark.parametrize("resolved", [False, True])
def test_the_load_time_decision_budgets_an_unresolved_encoder_dense(
fake_runtime, monkeypatch, resolved
):
backend, _spy = _load_backend(monkeypatch)
_uncompilable(monkeypatch)
heard: list = []
real = DiffusionBackend._bf16_resident_plan
def _spy_plan(self, plan, *a, **k):
heard.append(k.get("text_encoder_quant"))
return real(self, plan, *a, **k)
monkeypatch.setattr(DiffusionBackend, "_bf16_resident_plan", _spy_plan)
monkeypatch.setattr(dmod, "resolve_te_quant_request", lambda *_a, **_k: ("fp8", False))
_load(
backend,
_pipeline_prequant_planned = None,
_pipeline_prequant_skipped = (),
_te_prequant_resolved = resolved,
)
assert heard == [("fp8" if resolved else None)]
def test_the_kept_bf16_weights_are_placed_by_the_plan_that_proved_the_fit(
fake_runtime, monkeypatch
):
backend, spy = _load_backend(monkeypatch)
_uncompilable(monkeypatch)
real_plan = DiffusionBackend._plan_memory
def _plan(self, *a, **k):
plan = real_plan(self, *a, **k)
if k.get("transformer_resident_override_mib") is None:
plan.offload_policy = "sequential"
return plan
monkeypatch.setattr(DiffusionBackend, "_plan_memory", _plan)
monkeypatch.setattr(
DiffusionBackend,
"_resident_sized_plan",
lambda _s, plan, *_a, **_k: types.SimpleNamespace(**vars(plan)),
)
placed: list = []
monkeypatch.setattr(
dmod,
"apply_memory_plan",
lambda _pipe, plan, *_a, **_k: placed.append(plan.offload_policy) or ("none", False),
)
_load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())
assert spy.quantised == []
assert placed == ["none"]
@pytest.mark.parametrize("outgoing_host_mib, expected", [(0, False), (10_000, True)])
def test_the_streamed_seed_credits_host_ram_the_outgoing_pipeline_frees(
monkeypatch, outgoing_host_mib, expected
):
# the plan runs before the old pipeline unloads; its host weights must not veto the pin
from core.inference import diffusion_memory
monkeypatch.setattr(diffusion_memory, "_installed_torchao_version", lambda: (0, 18))
monkeypatch.setattr(diffusion_memory, "_pinned_memory_capped", lambda: False)
monkeypatch.delenv(diffusion_memory.GROUP_OFFLOAD_PIN_ENV, raising = False)
monkeypatch.setattr(diffusion_memory, "_pin_budget_mib", lambda: 4_000)
monkeypatch.setattr(dmod, "pipeline_host_mib", lambda pipe: outgoing_host_mib)
backend = _settle_backend(monkeypatch)
backend._state = types.SimpleNamespace(pipe = object())
monkeypatch.setattr(
DiffusionBackend,
"_plan_memory",
lambda *_a, **_k: types.SimpleNamespace(
offload_policy = "group",
stream_transformer = True,
estimates = {"model_dense_mib": 20_000, "companion_dense_mib": 8_000},
),
)
assert (_settle(backend) == "fp8") is expected
def _small_dit_kept_bf16(monkeypatch, reason):
_measured_bf16_family(monkeypatch)
monkeypatch.setattr(dmod, "auto_bf16_when_resident_reason", lambda _name: reason)
monkeypatch.setattr(dmod, "compile_eligible", lambda *_a, **_k: True)
monkeypatch.setattr(dmod, "fp16_compile_explicit_only", lambda _t: False)
@pytest.mark.parametrize(
"reason, speed, expect_speed, expect_deferred",
[
(dmod.SMALL_DIT_BF16_WHEN_RESIDENT, None, "default", False),
(dmod.SMALL_DIT_BF16_WHEN_RESIDENT, "off", "off", False),
("measured: no faster", None, "off", True),
# Anima's reason keeps the deferred compile (background, from render 3).
(ANIMA_BF16_WHEN_RESIDENT, None, "off", True),
],
)
def test_a_small_dit_kept_bf16_compiles_from_render_1(
fake_runtime, monkeypatch, reason, speed, expect_speed, expect_deferred
):
backend, spy = _load_backend(monkeypatch)
_small_dit_kept_bf16(monkeypatch, reason)
status = _load(
backend, speed_mode = speed, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ()
)
assert spy.quantised == [] and spy.seeds == []
assert status["resolved"]["transformer_quant"]["value"] == "off"
assert backend._state.speed_mode == expect_speed
assert backend._state.speed_deferred is expect_deferred
@pytest.mark.parametrize(
"repo, kept",
[
("black-forest-labs/FLUX.2-klein-4B", True),
("black-forest-labs/FLUX.2-klein-base-4B", True),
("unsloth/FLUX.2-klein-4B", True),
("black-forest-labs/FLUX.2-klein-9B", False),
("black-forest-labs/FLUX.2-klein-base-9B", False),
("unsloth/FLUX.2-klein-9B", False),
("unsloth/FLUX.2-klein-base-9B", False),
],
)
def test_the_small_dit_rule_skips_the_9b_klein_variants(repo, kept):
fam = detect_family_for_pick(repo, None, None)
assert fam is not None and fam.name == "flux.2-klein"
assert (dmod._auto_keeps_bf16_reason(fam, "pipeline", repo) is not None) is kept
fits = types.SimpleNamespace(
offload_policy = "none",
estimates = {"safe_device_budget_mib": 60_000, "resident_required_mib": 30_000},
)
assert (dmod._auto_quant_eager_reason(fam, fits, None, "pipeline", repo) is not None) is kept
def test_a_large_variant_of_a_small_dit_family_quantises_under_auto(fake_runtime, monkeypatch):
backend, spy = _load_backend(monkeypatch)
_small_dit_kept_bf16(monkeypatch, dmod.SMALL_DIT_BF16_WHEN_RESIDENT)
seen = []
def excludes(base):
seen.append(base)
return base == Z_IMAGE_REPO
monkeypatch.setattr(dmod, "small_dit_rule_excludes", excludes)
_load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())
assert Z_IMAGE_REPO in seen # the load hands the decider its resolved base
assert spy.quantised == ["auto"]
@needs_torchao
@pytest.mark.parametrize("shared_repo", [False, True])
def test_denoiser_is_a_checkpoint_and_encoder_only_is_an_asset(monkeypatch, hub, shared_repo):
backend = _plan_backend(monkeypatch, planned = "fp8")
encoder_repo = PREQUANT_REPO if shared_repo else "test/shared-encoder"
monkeypatch.setattr(
DiffusionBackend,
"_te_prequant_plan_files",
lambda *_a, **_k: {
"text_encoder": (encoder_repo, [("text_encoder/model.safetensors", 1234)])
},
)
plan = backend.download_plan(Z_IMAGE_REPO, model_kind = "pipeline")
entries = {entry["repo_id"]: entry for entry in plan["entries"]}
assert entries[PREQUANT_REPO]["checkpoint"] is True
assert entries[PREQUANT_REPO]["hosted_denoiser"] is True
assert entries[Z_IMAGE_REPO]["hosted_denoiser"] is False
# The route returns this model, which would drop an undeclared field.
served = DiffusionDownloadPlanResponse(**plan).entries
assert {e.repo_id for e in served if e.hosted_denoiser} == {PREQUANT_REPO}
assert entries[encoder_repo]["checkpoint"] is shared_repo
assert entries[encoder_repo]["hosted_denoiser"] is shared_repo
assert "text_encoder/model.safetensors" in entries[encoder_repo]["files"]
assert entries[PREQUANT_REPO]["bytes"] == PREQUANT_BYTES + (1234 if shared_repo else 0)