Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio MLX images: plan text encoder and dense bits from the Metal working set (#14793)
Daniel Han572a66b4f284
diff --git a/studio/backend/core/inference/mlx_diffusion_backend.py b/studio/backend/core/inference/mlx_diffusion_backend.pyindex f2fcd582bfe..18ba28e3844 100644--- a/studio/backend/core/inference/mlx_diffusion_backend.py+++ b/studio/backend/core/inference/mlx_diffusion_backend.py@@ -132,6 +132,183 @@ def dense_bits(ram_bytes: Optional[int] = None) -> int:return 4 if 0 < ram <= _LOW_RAM_BYTES else 0+# Unified-memory plan. Rungs in order of accuracy cost, measured on Z-Image-Turbo (16 renders at 1024, LPIPS vs the+# all-bf16 MLX render, paired bootstrap): an 8-bit text encoder is within noise of bf16 (+0.002), a 4-bit one is not+# (+0.029, 13 of 16 images worse); re-quantizing the GGUF's own BF16 tensors to 4 bits costs +0.008. So the 4-bit+# text encoder is the last rung, taken only when nothing above it fits the Metal working set.+_TE_BIT_OPTIONS = (8, 4)+_DENSE_BIT_OPTIONS = (0, 8, 4)+# Group-64 affine: one bf16 scale and one bf16 bias per 64 weights.+_TE_GROUP_OVERHEAD_BITS = 2 * 16 / _TE_GROUP_SIZE+# Live activations above the weights during a step, and the VAE decode at 1024x1024 when it is tiled (512 px tiles).+ACTIVATION_HEADROOM_BYTES = 2 * 1024**3+# The untiled VAE decode at 1024x1024 above the weights (mflux Z-Image VAE, bf16, Apple M1 Metal): 6.64 GiB. Reserved+# whenever the runtime cannot tile the decode.+UNTILED_DECODE_HEADROOM_BYTES = int(6.64 * 1024**3)+_VAE_TILING_OFF = ("0", "off", "false", "no")+++def decode_headroom_bytes(runtime: Any) -> int:+ """Headroom for the decode path ``runtime`` takes: the tiled peak only when it can tile (``vae_tiled``) and tiling+ is not forced off, else the untiled 1024x1024 peak."""+ can_tile = callable(getattr(runtime, "vae_tiled", None))+ forced_off = (+ os.environ.get("UNSLOTH_DIFFUSION_MLX_VAE_TILING", "").strip().lower() in _VAE_TILING_OFF+ )+ if can_tile and not forced_off:+ return ACTIVATION_HEADROOM_BYTES+ return max(ACTIVATION_HEADROOM_BYTES, UNTILED_DECODE_HEADROOM_BYTES)+++@dataclass(frozen = True)+class QuantPlan:+ te_bits: Optional[int]+ dense_bits: int+ planned_bytes: int = 0+ budget_bytes: int = 0+ fits: bool = True+ source: str = "plan"++ def as_dict(self) -> dict[str, Any]:+ return {+ "te_bits": self.te_bits,+ "dense_bits": self.dense_bits,+ "planned_bytes": self.planned_bytes,+ "budget_bytes": self.budget_bytes,+ "fits": self.fits,+ "source": self.source,+ }+++def text_encoder_resident_bytes(+ bf16_bytes: int,+ bits: Optional[int],+ retained_bytes: int = 0,+) -> int:+ """``bf16_bytes`` of quantizable weights at ``bits`` plus ``retained_bytes`` (norms, biases, weights whose last+ dimension is not a multiple of the group size) left as they are."""+ if not bits:+ return int(bf16_bytes) + int(retained_bytes)+ return int(bf16_bytes * (bits + _TE_GROUP_OVERHEAD_BITS) / 16) + int(retained_bytes)+++def working_set_bytes(ram_bytes: Optional[int] = None) -> int:+ """Metal's recommendedMaxWorkingSetSize: what one process can keep resident before macOS starts paging it."""+ try:+ from utils.hardware.hardware import _metal_recommended_working_set_bytes+ recommended = int(_metal_recommended_working_set_bytes() or 0)+ except Exception: # noqa: BLE001 -- unreadable: estimate from RAM below+ recommended = 0+ if recommended > 0:+ return recommended+ ram = _total_ram_bytes() if ram_bytes is None else ram_bytes+ # Metal's own figure is about two thirds of RAM on 8-32 GB Macs.+ return int(ram * 2 / 3) if ram > 0 else 0+++def plan_quantization(+ *,+ budget_bytes: int,+ transformer_bytes: dict[int, int],+ text_encoder_bf16_bytes: int,+ vae_bytes: int = 0,+ headroom_bytes: int = ACTIVATION_HEADROOM_BYTES,+ text_encoder_retained_bytes: int = 0,+) -> QuantPlan:+ """The most accurate (text encoder bits, dense bits) whose resident set fits ``budget_bytes``; the smallest when+ none does. ``UNSLOTH_DIFFUSION_MLX_TE_BITS`` / ``_DENSE_BITS`` pin their axis."""+ forced_te = _env_int("UNSLOTH_DIFFUSION_MLX_TE_BITS", (4, 8, 16))+ forced_dense = _env_int("UNSLOTH_DIFFUSION_MLX_DENSE_BITS", (0, 4, 8))+ te_options = (forced_te,) if forced_te is not None else _TE_BIT_OPTIONS+ dense_options = (forced_dense,) if forced_dense is not None else _DENSE_BIT_OPTIONS+ rungs = [+ (None if te == 16 else te, dense)+ for te in te_options+ for dense in dense_options+ if dense in transformer_bytes+ ]+ if not rungs:+ raise ValueError("no converted transformer size for the requested dense bits")+ planned = None+ for te, dense in rungs:+ total = (+ transformer_bytes[dense]+ + text_encoder_resident_bytes(text_encoder_bf16_bytes, te, text_encoder_retained_bytes)+ + int(vae_bytes)+ + int(headroom_bytes)+ )+ planned = QuantPlan(te, dense, total, int(budget_bytes), total <= budget_bytes)+ if planned.fits:+ return planned+ return planned+++def _safetensors_numel(folder: Path) -> int:+ """Element count of every tensor in ``folder``'s safetensors, from the headers alone."""+ return sum(_safetensors_numel_split(folder))+++def _safetensors_numel_split(folder: Path) -> tuple[int, int]:+ """(quantizable, retained) element counts of ``folder``'s safetensors, from the headers alone. Quantizable is what+ ``MfluxRuntime.build`` can quantize: matrices whose last dimension is a multiple of the group size. Everything else+ (norms, biases, odd-width weights) stays at full precision."""+ import struct++ quantizable = 0+ retained = 0+ for path in sorted(Path(folder).glob("*.safetensors")):+ with open(path, "rb") as fh:+ (n,) = struct.unpack("<Q", fh.read(8))+ if n > 100_000_000:+ raise ValueError(f"{path.name}: implausible safetensors header")+ header = json.loads(fh.read(n))+ for name, info in header.items():+ if name == "__metadata__":+ continue+ shape = [int(dim) for dim in info.get("shape", ())]+ count = 1+ for dim in shape:+ count *= dim+ if len(shape) >= 2 and shape[-1] % _TE_GROUP_SIZE == 0:+ quantizable += count+ else:+ retained += count+ return quantizable, retained+++def plan_for_load(+ gguf_path: str,+ base_dir: str,+ *,+ headroom_bytes: int = ACTIVATION_HEADROOM_BYTES,+) -> QuantPlan:+ """The load's plan: header sizes of the GGUF and the base repo's text encoder / VAE against the working set.+ Any unreadable input falls back to the RAM-size rule (``text_encoder_bits`` / ``dense_bits``)."""+ try:+ sizes = convert.planned_output_bytes(gguf_path)+ # mflux holds the text encoder and VAE in bf16 (2 bytes per element) before any quantization.+ te_quantizable, te_retained = _safetensors_numel_split(Path(base_dir) / "text_encoder")+ te_bf16 = 2 * te_quantizable+ te_retained_bytes = 2 * te_retained+ vae = 2 * _safetensors_numel(Path(base_dir) / "vae")+ if te_bf16 + te_retained_bytes <= 0:+ raise ValueError("no text encoder weights found")+ budget = working_set_bytes()+ if budget <= 0:+ raise ValueError("Metal working set unknown")+ return plan_quantization(+ budget_bytes = budget,+ transformer_bytes = sizes,+ text_encoder_bf16_bytes = te_bf16,+ vae_bytes = vae,+ headroom_bytes = headroom_bytes,+ text_encoder_retained_bytes = te_retained_bytes,+ )+ except Exception as exc: # noqa: BLE001 -- the size-free rule below still loads+ logger.info("mlx_diffusion.quant_plan_fallback: %s", exc)+ return QuantPlan(text_encoder_bits(), dense_bits(), source = "ram_rule")++def _chip_name() -> str:try:out = subprocess.run(@@ -838,7 +1015,20 @@ class MlxDiffusionBackend:timings["fetch_s"] = round(time.perf_counter() - t0, 3)if _cancel_event.is_set():return- bits = dense_bits()+ plan = plan_for_load(+ gguf_path, base_dir, headroom_bytes = decode_headroom_bytes(self._rt())+ )+ bits = plan.dense_bits+ timings["quant_plan"] = plan.as_dict()+ if not plan.fits:+ logger.warning(+ "mlx_diffusion.quant_plan_over_budget: planned %.2f GiB > working set %.2f GiB "+ "even at te_bits=%s dense_bits=%s",+ plan.planned_bytes / 1024**3,+ plan.budget_bytes / 1024**3,+ plan.te_bits,+ plan.dense_bits,+ )revision = _snapshot_revision(gguf_path)if revision is None:st = os.stat(gguf_path)@@ -872,7 +1062,7 @@ class MlxDiffusionBackend:if _cancel_event.is_set():returnprecision = compute_precision_name()- te_bits = text_encoder_bits()+ te_bits = plan.te_bitst2 = time.perf_counter()runtime = self._rt()# Free the replaced model first: two resident DiTs do not fit a small Mac.diff --git a/studio/backend/core/inference/mlx_diffusion_convert.py b/studio/backend/core/inference/mlx_diffusion_convert.pyindex cf85584483b..ed22d4f767c 100644--- a/studio/backend/core/inference/mlx_diffusion_convert.py+++ b/studio/backend/core/inference/mlx_diffusion_convert.py@@ -528,6 +528,21 @@ def _header_bytes(entries: list[tuple[str, str, list[int]]], metadata: dict[str,return blob+def planned_output_bytes(+ gguf_path: os.PathLike | str, dense_bits_options: tuple[int, ...] = (0, 8, 4)+) -> dict[int, int]:+ """Exact size of the converted transformer per ``dense_bits``, from the GGUF header alone (no tensor data read)."""+ from gguf import GGUFReader++ reader = GGUFReader(str(gguf_path))+ _check_types(reader)+ sizes: dict[int, int] = {}+ for bits in dense_bits_options:+ jobs, _ = _plan(reader, dense_bits = bits, scale_dtype = "bf16")+ sizes[bits] = sum(_nbytes(dtype, shape) for job in jobs for _n, dtype, shape in job.outputs)+ return sizes++def convert_gguf_to_mflux(gguf_path: os.PathLike | str,out_dir: os.PathLike | str,diff --git a/studio/backend/tests/test_mlx_diffusion_backend.py b/studio/backend/tests/test_mlx_diffusion_backend.pyindex 29cf7b280f2..a3a9213eec8 100644--- a/studio/backend/tests/test_mlx_diffusion_backend.py+++ b/studio/backend/tests/test_mlx_diffusion_backend.py@@ -542,6 +542,219 @@ def test_a_local_undistilled_gguf_resolves_without_the_hub(tmp_path, monkeypatchassert resolve(str(tmp_path), fam, None, "z-image-turbo-Q4_K_M.gguf", fallback = BASE) == BASE+GIB = 1024**3+# Unsloth's Z-Image-Turbo Q4_0 GGUF converted at dense_bits 0 / 8 / 4, and the base repo's Qwen3-4B text encoder and+# VAE in bf16 (header sizes of the real files).+ZIMAGE_Q4_0 = {0: int(4.51 * GIB), 8: int(3.90 * GIB), 4: int(3.58 * GIB)}+ZIMAGE_TE_BF16 = int(7.5 * GIB)+ZIMAGE_VAE = int(0.16 * GIB)+++def _clear_quant_env(monkeypatch):+ for name in ("UNSLOTH_DIFFUSION_MLX_TE_BITS", "UNSLOTH_DIFFUSION_MLX_DENSE_BITS"):+ monkeypatch.delenv(name, raising = False)+++ "budget, expected, fits",+ [+ # 32 GB and 16 GB Macs (Metal working set 21.3 / 10.67 GiB): the 8-bit text encoder and the GGUF as stored.+ (int(21.3 * GIB), (8, 0), True),+ (int(10.67 * GIB), (8, 0), True),+ # Tighter budgets walk down the rungs: the dense tensors first, the text encoder last.+ (int(10.2 * GIB), (8, 8), True),+ (int(9.8 * GIB), (8, 4), True),+ (int(9.0 * GIB), (4, 0), True),+ (int(8.5 * GIB), (4, 8), True),+ (int(8.0 * GIB), (4, 4), True),+ # 8 GB Mac: nothing fits, so the smallest build (what the RAM rule picked).+ (int(5.33 * GIB), (4, 4), False),+ ],+)+def test_quant_plan_takes_the_most_accurate_build_that_fits(monkeypatch, budget, expected, fits):+ _clear_quant_env(monkeypatch)+ plan = mb.plan_quantization(+ budget_bytes = budget,+ transformer_bytes = ZIMAGE_Q4_0,+ text_encoder_bf16_bytes = ZIMAGE_TE_BF16,+ vae_bytes = ZIMAGE_VAE,+ )+ assert (plan.te_bits, plan.dense_bits) == expected+ assert plan.fits is fits and plan.budget_bytes == budget+ if fits:+ assert plan.planned_bytes <= budget+++def test_quant_plan_shrinks_monotonically_with_the_budget(monkeypatch):+ _clear_quant_env(monkeypatch)+ previous = None+ seen = set()+ for tenth in range(220, 40, -1):+ plan = mb.plan_quantization(+ budget_bytes = int(tenth / 10 * GIB),+ transformer_bytes = ZIMAGE_Q4_0,+ text_encoder_bf16_bytes = ZIMAGE_TE_BF16,+ vae_bytes = ZIMAGE_VAE,+ )+ if previous is not None:+ assert plan.planned_bytes <= previous+ previous = plan.planned_bytes+ seen.add((plan.te_bits, plan.dense_bits))+ assert seen == {(8, 0), (8, 8), (8, 4), (4, 0), (4, 8), (4, 4)}+++def test_quant_plan_env_pins_one_axis(monkeypatch):+ _clear_quant_env(monkeypatch)+ monkeypatch.setenv("UNSLOTH_DIFFUSION_MLX_TE_BITS", "16")+ plan = mb.plan_quantization(+ budget_bytes = int(10.67 * GIB),+ transformer_bytes = ZIMAGE_Q4_0,+ text_encoder_bf16_bytes = ZIMAGE_TE_BF16,+ vae_bytes = ZIMAGE_VAE,+ )+ assert plan.te_bits is None and plan.dense_bits == 4 and not plan.fits+ monkeypatch.setenv("UNSLOTH_DIFFUSION_MLX_TE_BITS", "4")+ monkeypatch.setenv("UNSLOTH_DIFFUSION_MLX_DENSE_BITS", "0")+ plan = mb.plan_quantization(+ budget_bytes = int(64 * GIB),+ transformer_bytes = ZIMAGE_Q4_0,+ text_encoder_bf16_bytes = ZIMAGE_TE_BF16,+ )+ assert (plan.te_bits, plan.dense_bits) == (4, 0)+++def _header_only_safetensors(path, shapes):+ import struct++ header = {"__metadata__": {"format": "pt"}}+ offset = 0+ for name, shape in shapes.items():+ size = 2+ for dim in shape:+ size *= dim+ header[name] = {+ "dtype": "BF16",+ "shape": list(shape),+ "data_offsets": [offset, offset + size],+ }+ offset += size+ blob = json.dumps(header).encode()+ path.write_bytes(struct.pack("<Q", len(blob)) + blob)+++def test_a_16gb_mac_loads_z_image_q4_0_with_an_8bit_text_encoder(env, monkeypatch, tmp_path):+ """main quantized the text encoder to 4 bits on every Mac up to 16 GB, which costs +0.03 LPIPS."""+ backend, runtime, calls = env+ _clear_quant_env(monkeypatch)+ base_dir = tmp_path / "base"+ # 3.75 Gi bf16 elements = the 7.5 GiB text encoder; 80 Mi = the VAE.+ _header_only_safetensors(+ base_dir / "text_encoder" / "model.safetensors", {"w": (3 * 1024, 1024, 1280)}+ )+ _header_only_safetensors(base_dir / "vae" / "model.safetensors", {"w": (80, 1024, 1024)})+ monkeypatch.setattr(mb.convert, "planned_output_bytes", lambda path, *a, **k: dict(ZIMAGE_Q4_0))+ monkeypatch.setattr(mb, "_total_ram_bytes", lambda: 16 * GIB)+ monkeypatch.setattr(mb, "working_set_bytes", lambda ram_bytes = None: int(10.67 * GIB))+ monkeypatch.delenv("UNSLOTH_DIFFUSION_MLX_VAE_TILING", raising = False)+ # A runtime that tiles the decode when it would not fit, so the tiled 2 GiB decode peak is what is reserved.+ monkeypatch.setattr(runtime, "vae_tiled", lambda: False, raising = False)+ assert _load(backend)["phase"] == "ready"+ assert runtime.builds[-1]["te_bits"] == 8+ assert calls["dense_bits"] == 0+ status = backend.status()+ assert status["text_encoder_quant"] == "mlx_int8"+ plan = status["mlx"]["timings"]["quant_plan"]+ assert plan["source"] == "plan" and plan["fits"] is True+ assert plan["planned_bytes"] <= plan["budget_bytes"] == int(10.67 * GIB)+++def test_a_runtime_without_tiled_decode_reserves_the_untiled_decode_peak(+ env, monkeypatch, tmp_path+):+ """The untiled 1024x1024 decode needs 6.64 GiB above the weights, not the tiled 2 GiB: on a 16 GB Mac nothing fits+ then, so the plan takes the smallest build instead of an 8-bit text encoder that pages at decode."""+ backend, runtime, calls = env+ _clear_quant_env(monkeypatch)+ monkeypatch.delenv("UNSLOTH_DIFFUSION_MLX_VAE_TILING", raising = False)+ assert not hasattr(runtime, "vae_tiled")+ base_dir = tmp_path / "base"+ _header_only_safetensors(+ base_dir / "text_encoder" / "model.safetensors", {"w": (3 * 1024, 1024, 1280)}+ )+ _header_only_safetensors(base_dir / "vae" / "model.safetensors", {"w": (80, 1024, 1024)})+ monkeypatch.setattr(mb.convert, "planned_output_bytes", lambda path, *a, **k: dict(ZIMAGE_Q4_0))+ monkeypatch.setattr(mb, "working_set_bytes", lambda ram_bytes = None: int(10.67 * GIB))+ assert _load(backend)["phase"] == "ready"+ plan = backend.status()["mlx"]["timings"]["quant_plan"]+ assert (plan["te_bits"], plan["dense_bits"]) == (4, 4) and plan["fits"] is False+ assert runtime.builds[-1]["te_bits"] == 4 and calls["dense_bits"] == 4+++def test_decode_headroom_follows_the_decode_path(monkeypatch):+ class _Tiles:+ def vae_tiled(self):+ return False++ monkeypatch.delenv("UNSLOTH_DIFFUSION_MLX_VAE_TILING", raising = False)+ assert mb.decode_headroom_bytes(object()) == mb.UNTILED_DECODE_HEADROOM_BYTES+ assert mb.decode_headroom_bytes(_Tiles()) == mb.ACTIVATION_HEADROOM_BYTES+ monkeypatch.setenv("UNSLOTH_DIFFUSION_MLX_VAE_TILING", "0")+ assert mb.decode_headroom_bytes(_Tiles()) == mb.UNTILED_DECODE_HEADROOM_BYTES+++def test_unquantizable_text_encoder_tensors_stay_full_size(monkeypatch, tmp_path):+ """Norms, biases and odd-width weights are not quantized by the runtime, so the plan keeps them at bf16."""+ _clear_quant_env(monkeypatch)+ folder = tmp_path / "text_encoder"+ folder.mkdir()+ _header_only_safetensors(+ folder / "model.safetensors",+ {"q.weight": (256, 128), "q.bias": (256,), "norm.weight": (128,), "odd.weight": (64, 100)},+ )+ quantizable, retained = mb._safetensors_numel_split(folder)+ assert quantizable == 256 * 128 and retained == 256 + 128 + 64 * 100+ assert mb._safetensors_numel(folder) == quantizable + retained+ # A text encoder that is half retained tensors: the 8-bit rung must not count those at 8 bits.+ half = int(2 * GIB)+ assert (+ mb.text_encoder_resident_bytes(half, 8, half)+ == mb.text_encoder_resident_bytes(half, 8) + half+ )+ budget = (+ ZIMAGE_Q4_0[0] + mb.text_encoder_resident_bytes(2 * half, 8) + mb.ACTIVATION_HEADROOM_BYTES+ )+ all_quantized = mb.plan_quantization(+ budget_bytes = budget, transformer_bytes = ZIMAGE_Q4_0, text_encoder_bf16_bytes = 2 * half+ )+ split = mb.plan_quantization(+ budget_bytes = budget,+ transformer_bytes = ZIMAGE_Q4_0,+ text_encoder_bf16_bytes = half,+ text_encoder_retained_bytes = half,+ )+ assert (all_quantized.te_bits, all_quantized.dense_bits) == (8, 0)+ assert (split.te_bits, split.dense_bits) != (8, 0)+ assert split.planned_bytes <= budget+++def test_an_unreadable_gguf_keeps_the_ram_rule(env, monkeypatch):+ backend, runtime, calls = env+ _clear_quant_env(monkeypatch)+ monkeypatch.setattr(mb, "_total_ram_bytes", lambda: 16 * GIB)+ assert _load(backend)["phase"] == "ready"+ assert runtime.builds[-1]["te_bits"] == 4 and calls["dense_bits"] == 4+ assert backend.status()["mlx"]["timings"]["quant_plan"]["source"] == "ram_rule"+++def test_working_set_falls_back_to_two_thirds_of_ram(monkeypatch):+ import utils.hardware.hardware as hw++ monkeypatch.setattr(hw, "_metal_recommended_working_set_bytes", lambda: 0)+ assert mb.working_set_bytes(12 * GIB) == 8 * GIB+ monkeypatch.setattr(hw, "_metal_recommended_working_set_bytes", lambda: 11 * GIB)+ assert mb.working_set_bytes(12 * GIB) == 11 * GIB++def test_policy_helpers(monkeypatch):for name in ("UNSLOTH_DIFFUSION_MLX_TE_BITS",diff --git a/studio/backend/tests/test_mlx_diffusion_convert.py b/studio/backend/tests/test_mlx_diffusion_convert.pyindex ca2401da911..9ebb1e82499 100644--- a/studio/backend/tests/test_mlx_diffusion_convert.py+++ b/studio/backend/tests/test_mlx_diffusion_convert.py@@ -409,6 +409,68 @@ def test_conversion_complete_detects_truncation_and_source_change(tmp_path):assert not conv.conversion_complete(tmp_path / "out", source_path = src)[email protected]("bits", [0, 8, 4])+def test_planned_output_bytes_is_the_converted_size(tmp_path, bits):+ src = tmp_path / "z-image-turbo-Q4_0.gguf"+ _write_gguf(src)+ planned = conv.planned_output_bytes(src)+ summary = conv.convert_gguf_to_mflux(src, tmp_path / f"out{bits}", dense_bits = bits)+ assert planned[bits] == summary["bytes"]+ assert planned[0] > planned[8] > planned[4]+++def test_planned_output_bytes_refuses_what_conversion_refuses(tmp_path):+ src = tmp_path / "z-image-turbo-Q3_K_M.gguf"++ def extra(w):+ raw = _rng(2).integers(0, 255, size = (4, 110), dtype = np.uint8)+ w.add_tensor("layers.0.feed_forward.w2.weight", raw, raw_dtype = T.Q3_K)++ _write_gguf(src, extra = extra)+ with pytest.raises(conv.UnsupportedGGUFError, match = "Q3_K"):+ conv.planned_output_bytes(src)+++def test_mflux_loads_a_mixed_4_and_8_bit_conversion_at_each_layers_bits(tmp_path):+ """The 8-bit dense rung beside 4-bit GGUF blocks: mflux must rebuild each layer at its stored bits."""+ mx = pytest.importorskip("mlx.core")+ nn = pytest.importorskip("mlx.nn")+ applier = pytest.importorskip("mflux.models.common.weights.loading.weight_applier")+ src = tmp_path / "z-image-turbo-Q4_0.gguf"+ _write_gguf(src)+ conv.convert_gguf_to_mflux(src, tmp_path / "out", dense_bits = 8)+ loaded = mx.load(str(tmp_path / "out" / "transformer" / "0.safetensors"))++ class Ref(nn.Module):+ def __init__(self, rows, cols):+ super().__init__()+ self.lin = nn.Linear(cols, rows, bias = False)++ layers = {+ "layers.0.attention.to_out.0": (64, 64, 4),+ "layers.0.feed_forward.w1": (96, 64, 8),+ "context_refiner.0.feed_forward.w2": (32, 64, 8),+ }+ x = mx.array(_rng(11).standard_normal((3, 64)).astype(np.float32))+ for path, (rows, cols, bits) in layers.items():+ module = nn.Linear(cols, rows, bias = False)+ layer_weights = {k: loaded[f"{path}.{k}"] for k in ("weight", "scales", "biases")}+ predicate = applier.WeightApplier._stored_layer_predicate({"m": layer_weights}, None)+ holder = Ref(rows, cols)+ holder.lin = module+ nn.quantize(holder, class_predicate = lambda p, m: predicate("m", m), bits = 4, group_size = 64)+ assert holder.lin.bits == bits and holder.lin.group_size == 32+ holder.lin.update(layer_weights)+ got = np.array(holder.lin(x).astype(mx.float32))+ ref_w = conv.dequantize_affine(+ np.array(layer_weights["weight"]),+ np.array(layer_weights["scales"].astype(mx.float32)),+ np.array(layer_weights["biases"].astype(mx.float32)),+ bits,+ )+ np.testing.assert_allclose(got, np.array(x) @ ref_w.T, rtol = 2e-2, atol = 2e-2)++def test_cache_dir_for(tmp_path, monkeypatch):monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))d = conv.cache_dir_for(