Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio MLX images: plan text encoder and dense bits from the Metal working set (#14793)

Daniel Han572a66b4f284
diff --git a/studio/backend/core/inference/mlx_diffusion_backend.py b/studio/backend/core/inference/mlx_diffusion_backend.py
index f2fcd582bfe..18ba28e3844 100644
--- a/studio/backend/core/inference/mlx_diffusion_backend.py
+++ b/studio/backend/core/inference/mlx_diffusion_backend.py
@@ -132,6 +132,183 @@ def dense_bits(ram_bytes: Optional[int] = None) -> int:
return 4 if 0 < ram <= _LOW_RAM_BYTES else 0
+# Unified-memory plan. Rungs in order of accuracy cost, measured on Z-Image-Turbo (16 renders at 1024, LPIPS vs the
+# all-bf16 MLX render, paired bootstrap): an 8-bit text encoder is within noise of bf16 (+0.002), a 4-bit one is not
+# (+0.029, 13 of 16 images worse); re-quantizing the GGUF's own BF16 tensors to 4 bits costs +0.008. So the 4-bit
+# text encoder is the last rung, taken only when nothing above it fits the Metal working set.
+_TE_BIT_OPTIONS = (8, 4)
+_DENSE_BIT_OPTIONS = (0, 8, 4)
+# Group-64 affine: one bf16 scale and one bf16 bias per 64 weights.
+_TE_GROUP_OVERHEAD_BITS = 2 * 16 / _TE_GROUP_SIZE
+# Live activations above the weights during a step, and the VAE decode at 1024x1024 when it is tiled (512 px tiles).
+ACTIVATION_HEADROOM_BYTES = 2 * 1024**3
+# The untiled VAE decode at 1024x1024 above the weights (mflux Z-Image VAE, bf16, Apple M1 Metal): 6.64 GiB. Reserved
+# whenever the runtime cannot tile the decode.
+UNTILED_DECODE_HEADROOM_BYTES = int(6.64 * 1024**3)
+_VAE_TILING_OFF = ("0", "off", "false", "no")
+
+
+def decode_headroom_bytes(runtime: Any) -> int:
+ """Headroom for the decode path ``runtime`` takes: the tiled peak only when it can tile (``vae_tiled``) and tiling
+ is not forced off, else the untiled 1024x1024 peak."""
+ can_tile = callable(getattr(runtime, "vae_tiled", None))
+ forced_off = (
+ os.environ.get("UNSLOTH_DIFFUSION_MLX_VAE_TILING", "").strip().lower() in _VAE_TILING_OFF
+ )
+ if can_tile and not forced_off:
+ return ACTIVATION_HEADROOM_BYTES
+ return max(ACTIVATION_HEADROOM_BYTES, UNTILED_DECODE_HEADROOM_BYTES)
+
+
+@dataclass(frozen = True)
+class QuantPlan:
+ te_bits: Optional[int]
+ dense_bits: int
+ planned_bytes: int = 0
+ budget_bytes: int = 0
+ fits: bool = True
+ source: str = "plan"
+
+ def as_dict(self) -> dict[str, Any]:
+ return {
+ "te_bits": self.te_bits,
+ "dense_bits": self.dense_bits,
+ "planned_bytes": self.planned_bytes,
+ "budget_bytes": self.budget_bytes,
+ "fits": self.fits,
+ "source": self.source,
+ }
+
+
+def text_encoder_resident_bytes(
+ bf16_bytes: int,
+ bits: Optional[int],
+ retained_bytes: int = 0,
+) -> int:
+ """``bf16_bytes`` of quantizable weights at ``bits`` plus ``retained_bytes`` (norms, biases, weights whose last
+ dimension is not a multiple of the group size) left as they are."""
+ if not bits:
+ return int(bf16_bytes) + int(retained_bytes)
+ return int(bf16_bytes * (bits + _TE_GROUP_OVERHEAD_BITS) / 16) + int(retained_bytes)
+
+
+def working_set_bytes(ram_bytes: Optional[int] = None) -> int:
+ """Metal's recommendedMaxWorkingSetSize: what one process can keep resident before macOS starts paging it."""
+ try:
+ from utils.hardware.hardware import _metal_recommended_working_set_bytes
+ recommended = int(_metal_recommended_working_set_bytes() or 0)
+ except Exception: # noqa: BLE001 -- unreadable: estimate from RAM below
+ recommended = 0
+ if recommended > 0:
+ return recommended
+ ram = _total_ram_bytes() if ram_bytes is None else ram_bytes
+ # Metal's own figure is about two thirds of RAM on 8-32 GB Macs.
+ return int(ram * 2 / 3) if ram > 0 else 0
+
+
+def plan_quantization(
+ *,
+ budget_bytes: int,
+ transformer_bytes: dict[int, int],
+ text_encoder_bf16_bytes: int,
+ vae_bytes: int = 0,
+ headroom_bytes: int = ACTIVATION_HEADROOM_BYTES,
+ text_encoder_retained_bytes: int = 0,
+) -> QuantPlan:
+ """The most accurate (text encoder bits, dense bits) whose resident set fits ``budget_bytes``; the smallest when
+ none does. ``UNSLOTH_DIFFUSION_MLX_TE_BITS`` / ``_DENSE_BITS`` pin their axis."""
+ forced_te = _env_int("UNSLOTH_DIFFUSION_MLX_TE_BITS", (4, 8, 16))
+ forced_dense = _env_int("UNSLOTH_DIFFUSION_MLX_DENSE_BITS", (0, 4, 8))
+ te_options = (forced_te,) if forced_te is not None else _TE_BIT_OPTIONS
+ dense_options = (forced_dense,) if forced_dense is not None else _DENSE_BIT_OPTIONS
+ rungs = [
+ (None if te == 16 else te, dense)
+ for te in te_options
+ for dense in dense_options
+ if dense in transformer_bytes
+ ]
+ if not rungs:
+ raise ValueError("no converted transformer size for the requested dense bits")
+ planned = None
+ for te, dense in rungs:
+ total = (
+ transformer_bytes[dense]
+ + text_encoder_resident_bytes(text_encoder_bf16_bytes, te, text_encoder_retained_bytes)
+ + int(vae_bytes)
+ + int(headroom_bytes)
+ )
+ planned = QuantPlan(te, dense, total, int(budget_bytes), total <= budget_bytes)
+ if planned.fits:
+ return planned
+ return planned
+
+
+def _safetensors_numel(folder: Path) -> int:
+ """Element count of every tensor in ``folder``'s safetensors, from the headers alone."""
+ return sum(_safetensors_numel_split(folder))
+
+
+def _safetensors_numel_split(folder: Path) -> tuple[int, int]:
+ """(quantizable, retained) element counts of ``folder``'s safetensors, from the headers alone. Quantizable is what
+ ``MfluxRuntime.build`` can quantize: matrices whose last dimension is a multiple of the group size. Everything else
+ (norms, biases, odd-width weights) stays at full precision."""
+ import struct
+
+ quantizable = 0
+ retained = 0
+ for path in sorted(Path(folder).glob("*.safetensors")):
+ with open(path, "rb") as fh:
+ (n,) = struct.unpack("<Q", fh.read(8))
+ if n > 100_000_000:
+ raise ValueError(f"{path.name}: implausible safetensors header")
+ header = json.loads(fh.read(n))
+ for name, info in header.items():
+ if name == "__metadata__":
+ continue
+ shape = [int(dim) for dim in info.get("shape", ())]
+ count = 1
+ for dim in shape:
+ count *= dim
+ if len(shape) >= 2 and shape[-1] % _TE_GROUP_SIZE == 0:
+ quantizable += count
+ else:
+ retained += count
+ return quantizable, retained
+
+
+def plan_for_load(
+ gguf_path: str,
+ base_dir: str,
+ *,
+ headroom_bytes: int = ACTIVATION_HEADROOM_BYTES,
+) -> QuantPlan:
+ """The load's plan: header sizes of the GGUF and the base repo's text encoder / VAE against the working set.
+ Any unreadable input falls back to the RAM-size rule (``text_encoder_bits`` / ``dense_bits``)."""
+ try:
+ sizes = convert.planned_output_bytes(gguf_path)
+ # mflux holds the text encoder and VAE in bf16 (2 bytes per element) before any quantization.
+ te_quantizable, te_retained = _safetensors_numel_split(Path(base_dir) / "text_encoder")
+ te_bf16 = 2 * te_quantizable
+ te_retained_bytes = 2 * te_retained
+ vae = 2 * _safetensors_numel(Path(base_dir) / "vae")
+ if te_bf16 + te_retained_bytes <= 0:
+ raise ValueError("no text encoder weights found")
+ budget = working_set_bytes()
+ if budget <= 0:
+ raise ValueError("Metal working set unknown")
+ return plan_quantization(
+ budget_bytes = budget,
+ transformer_bytes = sizes,
+ text_encoder_bf16_bytes = te_bf16,
+ vae_bytes = vae,
+ headroom_bytes = headroom_bytes,
+ text_encoder_retained_bytes = te_retained_bytes,
+ )
+ except Exception as exc: # noqa: BLE001 -- the size-free rule below still loads
+ logger.info("mlx_diffusion.quant_plan_fallback: %s", exc)
+ return QuantPlan(text_encoder_bits(), dense_bits(), source = "ram_rule")
+
+
def _chip_name() -> str:
try:
out = subprocess.run(
@@ -838,7 +1015,20 @@ class MlxDiffusionBackend:
timings["fetch_s"] = round(time.perf_counter() - t0, 3)
if _cancel_event.is_set():
return
- bits = dense_bits()
+ plan = plan_for_load(
+ gguf_path, base_dir, headroom_bytes = decode_headroom_bytes(self._rt())
+ )
+ bits = plan.dense_bits
+ timings["quant_plan"] = plan.as_dict()
+ if not plan.fits:
+ logger.warning(
+ "mlx_diffusion.quant_plan_over_budget: planned %.2f GiB > working set %.2f GiB "
+ "even at te_bits=%s dense_bits=%s",
+ plan.planned_bytes / 1024**3,
+ plan.budget_bytes / 1024**3,
+ plan.te_bits,
+ plan.dense_bits,
+ )
revision = _snapshot_revision(gguf_path)
if revision is None:
st = os.stat(gguf_path)
@@ -872,7 +1062,7 @@ class MlxDiffusionBackend:
if _cancel_event.is_set():
return
precision = compute_precision_name()
- te_bits = text_encoder_bits()
+ te_bits = plan.te_bits
t2 = time.perf_counter()
runtime = self._rt()
# Free the replaced model first: two resident DiTs do not fit a small Mac.
diff --git a/studio/backend/core/inference/mlx_diffusion_convert.py b/studio/backend/core/inference/mlx_diffusion_convert.py
index cf85584483b..ed22d4f767c 100644
--- a/studio/backend/core/inference/mlx_diffusion_convert.py
+++ b/studio/backend/core/inference/mlx_diffusion_convert.py
@@ -528,6 +528,21 @@ def _header_bytes(entries: list[tuple[str, str, list[int]]], metadata: dict[str,
return blob
+def planned_output_bytes(
+ gguf_path: os.PathLike | str, dense_bits_options: tuple[int, ...] = (0, 8, 4)
+) -> dict[int, int]:
+ """Exact size of the converted transformer per ``dense_bits``, from the GGUF header alone (no tensor data read)."""
+ from gguf import GGUFReader
+
+ reader = GGUFReader(str(gguf_path))
+ _check_types(reader)
+ sizes: dict[int, int] = {}
+ for bits in dense_bits_options:
+ jobs, _ = _plan(reader, dense_bits = bits, scale_dtype = "bf16")
+ sizes[bits] = sum(_nbytes(dtype, shape) for job in jobs for _n, dtype, shape in job.outputs)
+ return sizes
+
+
def convert_gguf_to_mflux(
gguf_path: os.PathLike | str,
out_dir: os.PathLike | str,
diff --git a/studio/backend/tests/test_mlx_diffusion_backend.py b/studio/backend/tests/test_mlx_diffusion_backend.py
index 29cf7b280f2..a3a9213eec8 100644
--- a/studio/backend/tests/test_mlx_diffusion_backend.py
+++ b/studio/backend/tests/test_mlx_diffusion_backend.py
@@ -542,6 +542,219 @@ def test_a_local_undistilled_gguf_resolves_without_the_hub(tmp_path, monkeypatch
assert resolve(str(tmp_path), fam, None, "z-image-turbo-Q4_K_M.gguf", fallback = BASE) == BASE
+GIB = 1024**3
+# Unsloth's Z-Image-Turbo Q4_0 GGUF converted at dense_bits 0 / 8 / 4, and the base repo's Qwen3-4B text encoder and
+# VAE in bf16 (header sizes of the real files).
+ZIMAGE_Q4_0 = {0: int(4.51 * GIB), 8: int(3.90 * GIB), 4: int(3.58 * GIB)}
+ZIMAGE_TE_BF16 = int(7.5 * GIB)
+ZIMAGE_VAE = int(0.16 * GIB)
+
+
+def _clear_quant_env(monkeypatch):
+ for name in ("UNSLOTH_DIFFUSION_MLX_TE_BITS", "UNSLOTH_DIFFUSION_MLX_DENSE_BITS"):
+ monkeypatch.delenv(name, raising = False)
+
+
+ "budget, expected, fits",
+ [
+ # 32 GB and 16 GB Macs (Metal working set 21.3 / 10.67 GiB): the 8-bit text encoder and the GGUF as stored.
+ (int(21.3 * GIB), (8, 0), True),
+ (int(10.67 * GIB), (8, 0), True),
+ # Tighter budgets walk down the rungs: the dense tensors first, the text encoder last.
+ (int(10.2 * GIB), (8, 8), True),
+ (int(9.8 * GIB), (8, 4), True),
+ (int(9.0 * GIB), (4, 0), True),
+ (int(8.5 * GIB), (4, 8), True),
+ (int(8.0 * GIB), (4, 4), True),
+ # 8 GB Mac: nothing fits, so the smallest build (what the RAM rule picked).
+ (int(5.33 * GIB), (4, 4), False),
+ ],
+)
+def test_quant_plan_takes_the_most_accurate_build_that_fits(monkeypatch, budget, expected, fits):
+ _clear_quant_env(monkeypatch)
+ plan = mb.plan_quantization(
+ budget_bytes = budget,
+ transformer_bytes = ZIMAGE_Q4_0,
+ text_encoder_bf16_bytes = ZIMAGE_TE_BF16,
+ vae_bytes = ZIMAGE_VAE,
+ )
+ assert (plan.te_bits, plan.dense_bits) == expected
+ assert plan.fits is fits and plan.budget_bytes == budget
+ if fits:
+ assert plan.planned_bytes <= budget
+
+
+def test_quant_plan_shrinks_monotonically_with_the_budget(monkeypatch):
+ _clear_quant_env(monkeypatch)
+ previous = None
+ seen = set()
+ for tenth in range(220, 40, -1):
+ plan = mb.plan_quantization(
+ budget_bytes = int(tenth / 10 * GIB),
+ transformer_bytes = ZIMAGE_Q4_0,
+ text_encoder_bf16_bytes = ZIMAGE_TE_BF16,
+ vae_bytes = ZIMAGE_VAE,
+ )
+ if previous is not None:
+ assert plan.planned_bytes <= previous
+ previous = plan.planned_bytes
+ seen.add((plan.te_bits, plan.dense_bits))
+ assert seen == {(8, 0), (8, 8), (8, 4), (4, 0), (4, 8), (4, 4)}
+
+
+def test_quant_plan_env_pins_one_axis(monkeypatch):
+ _clear_quant_env(monkeypatch)
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_MLX_TE_BITS", "16")
+ plan = mb.plan_quantization(
+ budget_bytes = int(10.67 * GIB),
+ transformer_bytes = ZIMAGE_Q4_0,
+ text_encoder_bf16_bytes = ZIMAGE_TE_BF16,
+ vae_bytes = ZIMAGE_VAE,
+ )
+ assert plan.te_bits is None and plan.dense_bits == 4 and not plan.fits
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_MLX_TE_BITS", "4")
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_MLX_DENSE_BITS", "0")
+ plan = mb.plan_quantization(
+ budget_bytes = int(64 * GIB),
+ transformer_bytes = ZIMAGE_Q4_0,
+ text_encoder_bf16_bytes = ZIMAGE_TE_BF16,
+ )
+ assert (plan.te_bits, plan.dense_bits) == (4, 0)
+
+
+def _header_only_safetensors(path, shapes):
+ import struct
+
+ header = {"__metadata__": {"format": "pt"}}
+ offset = 0
+ for name, shape in shapes.items():
+ size = 2
+ for dim in shape:
+ size *= dim
+ header[name] = {
+ "dtype": "BF16",
+ "shape": list(shape),
+ "data_offsets": [offset, offset + size],
+ }
+ offset += size
+ blob = json.dumps(header).encode()
+ path.write_bytes(struct.pack("<Q", len(blob)) + blob)
+
+
+def test_a_16gb_mac_loads_z_image_q4_0_with_an_8bit_text_encoder(env, monkeypatch, tmp_path):
+ """main quantized the text encoder to 4 bits on every Mac up to 16 GB, which costs +0.03 LPIPS."""
+ backend, runtime, calls = env
+ _clear_quant_env(monkeypatch)
+ base_dir = tmp_path / "base"
+ # 3.75 Gi bf16 elements = the 7.5 GiB text encoder; 80 Mi = the VAE.
+ _header_only_safetensors(
+ base_dir / "text_encoder" / "model.safetensors", {"w": (3 * 1024, 1024, 1280)}
+ )
+ _header_only_safetensors(base_dir / "vae" / "model.safetensors", {"w": (80, 1024, 1024)})
+ monkeypatch.setattr(mb.convert, "planned_output_bytes", lambda path, *a, **k: dict(ZIMAGE_Q4_0))
+ monkeypatch.setattr(mb, "_total_ram_bytes", lambda: 16 * GIB)
+ monkeypatch.setattr(mb, "working_set_bytes", lambda ram_bytes = None: int(10.67 * GIB))
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_MLX_VAE_TILING", raising = False)
+ # A runtime that tiles the decode when it would not fit, so the tiled 2 GiB decode peak is what is reserved.
+ monkeypatch.setattr(runtime, "vae_tiled", lambda: False, raising = False)
+ assert _load(backend)["phase"] == "ready"
+ assert runtime.builds[-1]["te_bits"] == 8
+ assert calls["dense_bits"] == 0
+ status = backend.status()
+ assert status["text_encoder_quant"] == "mlx_int8"
+ plan = status["mlx"]["timings"]["quant_plan"]
+ assert plan["source"] == "plan" and plan["fits"] is True
+ assert plan["planned_bytes"] <= plan["budget_bytes"] == int(10.67 * GIB)
+
+
+def test_a_runtime_without_tiled_decode_reserves_the_untiled_decode_peak(
+ env, monkeypatch, tmp_path
+):
+ """The untiled 1024x1024 decode needs 6.64 GiB above the weights, not the tiled 2 GiB: on a 16 GB Mac nothing fits
+ then, so the plan takes the smallest build instead of an 8-bit text encoder that pages at decode."""
+ backend, runtime, calls = env
+ _clear_quant_env(monkeypatch)
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_MLX_VAE_TILING", raising = False)
+ assert not hasattr(runtime, "vae_tiled")
+ base_dir = tmp_path / "base"
+ _header_only_safetensors(
+ base_dir / "text_encoder" / "model.safetensors", {"w": (3 * 1024, 1024, 1280)}
+ )
+ _header_only_safetensors(base_dir / "vae" / "model.safetensors", {"w": (80, 1024, 1024)})
+ monkeypatch.setattr(mb.convert, "planned_output_bytes", lambda path, *a, **k: dict(ZIMAGE_Q4_0))
+ monkeypatch.setattr(mb, "working_set_bytes", lambda ram_bytes = None: int(10.67 * GIB))
+ assert _load(backend)["phase"] == "ready"
+ plan = backend.status()["mlx"]["timings"]["quant_plan"]
+ assert (plan["te_bits"], plan["dense_bits"]) == (4, 4) and plan["fits"] is False
+ assert runtime.builds[-1]["te_bits"] == 4 and calls["dense_bits"] == 4
+
+
+def test_decode_headroom_follows_the_decode_path(monkeypatch):
+ class _Tiles:
+ def vae_tiled(self):
+ return False
+
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_MLX_VAE_TILING", raising = False)
+ assert mb.decode_headroom_bytes(object()) == mb.UNTILED_DECODE_HEADROOM_BYTES
+ assert mb.decode_headroom_bytes(_Tiles()) == mb.ACTIVATION_HEADROOM_BYTES
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_MLX_VAE_TILING", "0")
+ assert mb.decode_headroom_bytes(_Tiles()) == mb.UNTILED_DECODE_HEADROOM_BYTES
+
+
+def test_unquantizable_text_encoder_tensors_stay_full_size(monkeypatch, tmp_path):
+ """Norms, biases and odd-width weights are not quantized by the runtime, so the plan keeps them at bf16."""
+ _clear_quant_env(monkeypatch)
+ folder = tmp_path / "text_encoder"
+ folder.mkdir()
+ _header_only_safetensors(
+ folder / "model.safetensors",
+ {"q.weight": (256, 128), "q.bias": (256,), "norm.weight": (128,), "odd.weight": (64, 100)},
+ )
+ quantizable, retained = mb._safetensors_numel_split(folder)
+ assert quantizable == 256 * 128 and retained == 256 + 128 + 64 * 100
+ assert mb._safetensors_numel(folder) == quantizable + retained
+ # A text encoder that is half retained tensors: the 8-bit rung must not count those at 8 bits.
+ half = int(2 * GIB)
+ assert (
+ mb.text_encoder_resident_bytes(half, 8, half)
+ == mb.text_encoder_resident_bytes(half, 8) + half
+ )
+ budget = (
+ ZIMAGE_Q4_0[0] + mb.text_encoder_resident_bytes(2 * half, 8) + mb.ACTIVATION_HEADROOM_BYTES
+ )
+ all_quantized = mb.plan_quantization(
+ budget_bytes = budget, transformer_bytes = ZIMAGE_Q4_0, text_encoder_bf16_bytes = 2 * half
+ )
+ split = mb.plan_quantization(
+ budget_bytes = budget,
+ transformer_bytes = ZIMAGE_Q4_0,
+ text_encoder_bf16_bytes = half,
+ text_encoder_retained_bytes = half,
+ )
+ assert (all_quantized.te_bits, all_quantized.dense_bits) == (8, 0)
+ assert (split.te_bits, split.dense_bits) != (8, 0)
+ assert split.planned_bytes <= budget
+
+
+def test_an_unreadable_gguf_keeps_the_ram_rule(env, monkeypatch):
+ backend, runtime, calls = env
+ _clear_quant_env(monkeypatch)
+ monkeypatch.setattr(mb, "_total_ram_bytes", lambda: 16 * GIB)
+ assert _load(backend)["phase"] == "ready"
+ assert runtime.builds[-1]["te_bits"] == 4 and calls["dense_bits"] == 4
+ assert backend.status()["mlx"]["timings"]["quant_plan"]["source"] == "ram_rule"
+
+
+def test_working_set_falls_back_to_two_thirds_of_ram(monkeypatch):
+ import utils.hardware.hardware as hw
+
+ monkeypatch.setattr(hw, "_metal_recommended_working_set_bytes", lambda: 0)
+ assert mb.working_set_bytes(12 * GIB) == 8 * GIB
+ monkeypatch.setattr(hw, "_metal_recommended_working_set_bytes", lambda: 11 * GIB)
+ assert mb.working_set_bytes(12 * GIB) == 11 * GIB
+
+
def test_policy_helpers(monkeypatch):
for name in (
"UNSLOTH_DIFFUSION_MLX_TE_BITS",
diff --git a/studio/backend/tests/test_mlx_diffusion_convert.py b/studio/backend/tests/test_mlx_diffusion_convert.py
index ca2401da911..9ebb1e82499 100644
--- a/studio/backend/tests/test_mlx_diffusion_convert.py
+++ b/studio/backend/tests/test_mlx_diffusion_convert.py
@@ -409,6 +409,68 @@ def test_conversion_complete_detects_truncation_and_source_change(tmp_path):
assert not conv.conversion_complete(tmp_path / "out", source_path = src)
[email protected]("bits", [0, 8, 4])
+def test_planned_output_bytes_is_the_converted_size(tmp_path, bits):
+ src = tmp_path / "z-image-turbo-Q4_0.gguf"
+ _write_gguf(src)
+ planned = conv.planned_output_bytes(src)
+ summary = conv.convert_gguf_to_mflux(src, tmp_path / f"out{bits}", dense_bits = bits)
+ assert planned[bits] == summary["bytes"]
+ assert planned[0] > planned[8] > planned[4]
+
+
+def test_planned_output_bytes_refuses_what_conversion_refuses(tmp_path):
+ src = tmp_path / "z-image-turbo-Q3_K_M.gguf"
+
+ def extra(w):
+ raw = _rng(2).integers(0, 255, size = (4, 110), dtype = np.uint8)
+ w.add_tensor("layers.0.feed_forward.w2.weight", raw, raw_dtype = T.Q3_K)
+
+ _write_gguf(src, extra = extra)
+ with pytest.raises(conv.UnsupportedGGUFError, match = "Q3_K"):
+ conv.planned_output_bytes(src)
+
+
+def test_mflux_loads_a_mixed_4_and_8_bit_conversion_at_each_layers_bits(tmp_path):
+ """The 8-bit dense rung beside 4-bit GGUF blocks: mflux must rebuild each layer at its stored bits."""
+ mx = pytest.importorskip("mlx.core")
+ nn = pytest.importorskip("mlx.nn")
+ applier = pytest.importorskip("mflux.models.common.weights.loading.weight_applier")
+ src = tmp_path / "z-image-turbo-Q4_0.gguf"
+ _write_gguf(src)
+ conv.convert_gguf_to_mflux(src, tmp_path / "out", dense_bits = 8)
+ loaded = mx.load(str(tmp_path / "out" / "transformer" / "0.safetensors"))
+
+ class Ref(nn.Module):
+ def __init__(self, rows, cols):
+ super().__init__()
+ self.lin = nn.Linear(cols, rows, bias = False)
+
+ layers = {
+ "layers.0.attention.to_out.0": (64, 64, 4),
+ "layers.0.feed_forward.w1": (96, 64, 8),
+ "context_refiner.0.feed_forward.w2": (32, 64, 8),
+ }
+ x = mx.array(_rng(11).standard_normal((3, 64)).astype(np.float32))
+ for path, (rows, cols, bits) in layers.items():
+ module = nn.Linear(cols, rows, bias = False)
+ layer_weights = {k: loaded[f"{path}.{k}"] for k in ("weight", "scales", "biases")}
+ predicate = applier.WeightApplier._stored_layer_predicate({"m": layer_weights}, None)
+ holder = Ref(rows, cols)
+ holder.lin = module
+ nn.quantize(holder, class_predicate = lambda p, m: predicate("m", m), bits = 4, group_size = 64)
+ assert holder.lin.bits == bits and holder.lin.group_size == 32
+ holder.lin.update(layer_weights)
+ got = np.array(holder.lin(x).astype(mx.float32))
+ ref_w = conv.dequantize_affine(
+ np.array(layer_weights["weight"]),
+ np.array(layer_weights["scales"].astype(mx.float32)),
+ np.array(layer_weights["biases"].astype(mx.float32)),
+ bits,
+ )
+ np.testing.assert_allclose(got, np.array(x) @ ref_w.T, rtol = 2e-2, atol = 2e-2)
+
+
def test_cache_dir_for(tmp_path, monkeypatch):
monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
d = conv.cache_dir_for(