Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: cache image native int8 builds in the local prequant cache (#15283)

Daniel Han48a9c8dd7090
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index eda31a3b873..e9f4d360a83 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -8707,6 +8707,12 @@ class DiffusionBackend:
native_offload_forced = native_offload_scheme is not None and (
_memory_request_forces_offload(memory_mode, cpu_offload)
)
+ # The only build the native cache can serve back (MPS / ROCm set native_scheme up front).
+ native_cache_ok = (
+ native_offload_forced
+ and native_scheme is None
+ and getattr(target, "device", None) == "cuda"
+ )
if transformer_quant_pinned is not None and not dense_quant_supported_kind(kind):
transformer_quant_decline = dense_quant_unsupported_kind_reason(kind)
transformer_quant_decline_status = RESOLVED_UNSUPPORTED
@@ -9683,6 +9689,34 @@ class DiffusionBackend:
family = getattr(fam, "name", None),
logger = logger,
)
+ if (
+ pipeline_seed_scheme is None
+ and native_cache_ok
+ and not _has_active_lora(loras)
+ and not _pipeline_prequant_skipped
+ and "transformer" not in pipe_kwargs
+ ):
+ native_cached = self._native_quant_from_cache(
+ fam,
+ transformer_cls,
+ native_offload_scheme,
+ target,
+ base = base,
+ source_base = _base_local_dir or fetch_base,
+ config_kwargs = {
+ "local_files_only": local_files_only,
+ "cache_dir": hub_cache_dir(),
+ **({"token": hf_token} if hf_token else {}),
+ },
+ cancel_check = lambda: self._raise_if_load_cancelled(
+ _load_token
+ ),
+ )
+ if native_cached is not None:
+ # Cached native build: skips loading dense shards and re-quantizing.
+ pipe_kwargs["transformer"] = native_cached
+ native_scheme = native_offload_scheme
+ transformer_quant_engaged = native_offload_scheme
if pipeline_seed_scheme is None and _pipeline_prequant_skipped:
# from_pretrained cannot re-fetch these from a local snapshot dir, so top them up here.
_base_local_dir = self._restore_skipped_transformer_shards(
@@ -10315,6 +10349,7 @@ class DiffusionBackend:
_load_token
),
defer = True,
+ native_cache = native_cache_ok,
)
if not hasattr(pending_cache_write, "start"):
pending_cache_write = None
@@ -11782,6 +11817,50 @@ class DiffusionBackend:
logger.warning("diffusion.prequant_cache: read skipped (%s)", exc)
return None
+ @staticmethod
+ def _native_quant_from_cache(
+ fam: Any,
+ transformer_cls: Any,
+ scheme: Optional[str],
+ target: Any,
+ *,
+ base: Optional[str],
+ source_base: Optional[str],
+ config_kwargs: Optional[dict] = None,
+ cancel_check: Optional[Callable[[], None]] = None,
+ ) -> Any:
+ """The forced-offload native ``transformer`` from the local prequant cache, or None. CUDA only. Never raises."""
+ try:
+ if fam is None or scheme is None or getattr(target, "device", None) != "cuda":
+ return None
+ from .diffusion_native_cache import native_spec, read_for_load
+
+ spec = native_spec(
+ scheme,
+ getattr(fam, "name", None),
+ act_int8 = scheme == TQ_INT8 and native_int8_act(target),
+ )
+ module = read_for_load(
+ fam,
+ spec,
+ transformer_cls,
+ base_repo = base,
+ source_base = source_base,
+ config_kwargs = config_kwargs,
+ logger = logger,
+ cancel_check = cancel_check,
+ )
+ if module is not None:
+ logger.info(
+ "diffusion.transformer_quant: native %s read from the local prequant cache "
+ "(the dense transformer is not loaded, no re-quantize)",
+ scheme,
+ )
+ return module
+ except Exception as exc: # noqa: BLE001 - a cache problem is a miss
+ logger.warning("diffusion.prequant_cache: native read skipped (%s)", exc)
+ return None
+
@staticmethod
def _save_pipeline_quant_to_cache(
pipe: Any,
@@ -11793,13 +11872,45 @@ class DiffusionBackend:
source_base: Optional[str],
cancel_check: Optional[Callable[[], None]] = None,
defer: bool = False,
+ native_cache: bool = False,
) -> Any:
"""Cache a load-time quantized ``transformer`` (single denoiser only, the cache keys that component; no LoRA).
- ``defer`` may return a ``PendingWrite`` for the caller to start once the load is published. Never raises."""
+ ``defer`` may return a ``PendingWrite`` for the caller to start once the load is published. A native build is
+ written only when ``native_cache`` says the load could read it back. Never raises."""
try:
names = [attr for attr, _module in (denoisers or ())]
if fam is None or names != ["transformer"] or _has_active_lora(loras):
return None
+ transformer = getattr(pipe, "transformer", None)
+ if (getattr(transformer, "_unsloth_runtime_quant_info", None) or {}).get("native"):
+ if not native_cache:
+ return None
+ from .diffusion_native_cache import save_deferred
+
+ if not defer:
+ from .diffusion_native_cache import build_spec, save
+ spec = build_spec(transformer, getattr(fam, "name", None))
+ return (
+ None
+ if spec is None
+ else save(
+ transformer,
+ fam,
+ spec,
+ base_repo = base,
+ source_base = source_base,
+ logger = logger,
+ cancel_check = cancel_check,
+ )
+ )
+ return save_deferred(
+ transformer,
+ fam,
+ base_repo = base,
+ source_base = source_base,
+ logger = logger,
+ cancel_check = cancel_check,
+ )
from .diffusion_prequant_cache import save_runtime_quant
return save_runtime_quant(
diff --git a/studio/backend/core/inference/diffusion_native_cache.py b/studio/backend/core/inference/diffusion_native_cache.py
index 454f3871f5f..3f4b1420461 100644
--- a/studio/backend/core/inference/diffusion_native_cache.py
+++ b/studio/backend/core/inference/diffusion_native_cache.py
@@ -454,3 +454,183 @@ def load(
except Exception: # noqa: BLE001
pass
return None
+
+
+def build_spec(transformer: Any, family: Optional[str]) -> Optional[dict]:
+ """``native_spec`` of an in-memory native build, read from its layers, or None (no native layers, or a mix)."""
+ try:
+ from .diffusion_native_quant import is_native_linear
+
+ schemes, acts = set(), set()
+ for mod in transformer.modules():
+ if is_native_linear(mod):
+ schemes.add(mod.scheme)
+ acts.add(bool(mod.act_int8))
+ if len(schemes) != 1 or len(acts) != 1:
+ return None
+ return native_spec(next(iter(schemes)), family, act_int8 = next(iter(acts)))
+ except Exception: # noqa: BLE001
+ return None
+
+
+class NativePendingWrite:
+ """Background ``save``, awaited by ``finish_background_writes``; writes live host tensors (no snapshot copy)."""
+
+ def __init__(
+ self,
+ run: Any,
+ key: str,
+ logger: Any = None,
+ ) -> None:
+ import threading
+
+ self._run_fn = run
+ self.key = key
+ self._logger = logger
+ self._cancel = threading.Event()
+ self._finished = threading.Event()
+ self._thread: Optional[Any] = None
+ self.result: Optional[str] = None
+ with _cache._lock:
+ _cache._BACKGROUND.add(self)
+
+ def _check(self) -> None:
+ if self._cancel.is_set():
+ raise RuntimeError("native cache write cancelled")
+
+ def start(self) -> bool:
+ import threading
+
+ with _cache._lock:
+ if self._thread is not None or self._run_fn is None or self._cancel.is_set():
+ return False
+ self._thread = threading.Thread(
+ target = self._run, name = f"native-cache-write-{self.key[:8]}", daemon = True
+ )
+ try:
+ self._thread.start()
+ except Exception: # noqa: BLE001 - no writer: the next load re-quantizes
+ with _cache._lock:
+ self._thread = None
+ self._close()
+ return False
+ return True
+
+ def _run(self) -> None:
+ try:
+ if self._run_fn is not None:
+ self.result = self._run_fn(self._check)
+ except Exception as exc: # noqa: BLE001 - a background cache write never surfaces
+ _cache._log(self._logger, "warning", f"native: background save failed: {exc}")
+ finally:
+ self._close()
+
+ def _close(self) -> None:
+ self._run_fn = None
+ with _cache._lock:
+ _cache._BACKGROUND.discard(self)
+ self._finished.set()
+
+ def discard(self) -> None:
+ with _cache._lock:
+ started = self._thread is not None
+ self._cancel.set()
+ if not started:
+ self._close()
+
+ def cancel(self) -> None:
+ self.discard()
+
+ def done(self) -> bool:
+ return self._finished.is_set()
+
+ def wait(self, timeout: Optional[float] = None) -> bool:
+ return self._finished.wait(timeout)
+
+
+def save_deferred(
+ transformer: Any,
+ fam: Any,
+ *,
+ base_repo: Optional[str],
+ source_base: Optional[str] = None,
+ component: str = "transformer",
+ logger: Any = None,
+ cancel_check: Optional[Any] = None,
+) -> Any:
+ """A ``NativePendingWrite`` (start after publish), the sync ``save`` result, or None. Never raises."""
+ try:
+ if not _cache.enabled():
+ return None
+ spec = build_spec(transformer, getattr(fam, "name", None))
+ if spec is None:
+ _cache._log(logger, "info", f"native {component}: skipped (no single native recipe)")
+ return None
+ recorded_base = str(base_repo or getattr(fam, "base_repo", "") or "")
+ keyed = _key_for(fam, spec, source_base or base_repo, component, base_id = recorded_base)
+ if keyed is None:
+ _cache._log(
+ logger,
+ "info",
+ f"native {component}: skipped (dense source shards not found locally)",
+ )
+ return None
+
+ def run(check: Optional[Any] = None) -> Optional[str]:
+ def both() -> None:
+ if check is not None:
+ check()
+ if cancel_check is not None and check is None:
+ cancel_check()
+
+ return save(
+ transformer,
+ fam,
+ spec,
+ base_repo = base_repo,
+ source_base = source_base,
+ component = component,
+ logger = logger,
+ cancel_check = both,
+ )
+
+ if not _cache.async_enabled():
+ return run()
+ return NativePendingWrite(run, keyed[0], logger)
+ except Exception as exc: # noqa: BLE001
+ _cache._log(
+ logger, "warning", f"native {component}: save skipped ({type(exc).__name__}: {exc})"
+ )
+ return None
+
+
+def read_for_load(
+ fam: Any,
+ spec: Optional[dict],
+ transformer_cls: Any,
+ *,
+ base_repo: Optional[str],
+ source_base: Optional[str],
+ component: str = "transformer",
+ config_kwargs: Optional[dict] = None,
+ logger: Any = None,
+ cancel_check: Optional[Any] = None,
+) -> Any:
+ """``lookup`` + ``load`` of the cached native ``component``, or None (a miss). Never raises."""
+ if spec is None or transformer_cls is None:
+ return None
+ path = lookup(
+ fam, spec, base_repo = base_repo, source_base = source_base, component = component, logger = logger
+ )
+ if path is None:
+ return None
+ return load(
+ path,
+ transformer_cls,
+ str(source_base or base_repo),
+ component = component,
+ spec = spec,
+ config_kwargs = config_kwargs,
+ logger = logger,
+ cancel_check = cancel_check,
+ )
diff --git a/studio/backend/tests/test_diffusion_image_native_prequant_cache.py b/studio/backend/tests/test_diffusion_image_native_prequant_cache.py
new file mode 100644
index 00000000000..e50da16158c
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_image_native_prequant_cache.py
@@ -0,0 +1,265 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Image native int8 builds are cached in the background and read back instead of re-quantizing."""
+
+from __future__ import annotations
+
+import sys
+import types
+from pathlib import Path
+
+import pytest
+
+torch = pytest.importorskip("torch")
+diffusers = pytest.importorskip("diffusers")
+pytest.importorskip("safetensors")
+pytest.importorskip("accelerate")
+
+BACKEND = Path(__file__).resolve().parents[1]
+if str(BACKEND) not in sys.path:
+ sys.path.insert(0, str(BACKEND))
+
+from core.inference import diffusion as D # noqa: E402
+from core.inference import diffusion_native_cache as NC # noqa: E402
+from core.inference import diffusion_prequant_cache as C # noqa: E402
+from core.inference import diffusion_transformer_quant as tq # noqa: E402
+from core.inference.diffusion_families import detect_family # noqa: E402
+
+FAMILY = "flux.1"
+CUDA = types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16)
+
+
+def _fam():
+ return detect_family("black-forest-labs/FLUX.1-schnell", override = FAMILY)
+
+
+def base_dir(tmp_path):
+ root = tmp_path / "FLUX.1-schnell"
+ torch.manual_seed(0)
+ # 2 x 256 = 512 wide: every block Linear passes the 512 filter and g256 ConvRot divides it.
+ model = diffusers.FluxTransformer2DModel(
+ patch_size = 1,
+ in_channels = 16,
+ num_layers = 1,
+ num_single_layers = 1,
+ attention_head_dim = 256,
+ num_attention_heads = 2,
+ joint_attention_dim = 512,
+ pooled_projection_dim = 64,
+ axes_dims_rope = (32, 112, 112),
+ )
+ model.save_pretrained(str(root / "transformer"), safe_serialization = True)
+ return str(root)
+
+
[email protected](autouse = True)
+def cache_root(tmp_path, monkeypatch):
+ root = tmp_path / "cache"
+ monkeypatch.setenv(C.ENV_DIR, str(root))
+ monkeypatch.delenv(C.ENV_MODE, raising = False)
+ monkeypatch.delenv(C.ENV_ASYNC, raising = False)
+ monkeypatch.setattr(C, "_hub_source_stale", lambda shards, logger = None: False)
+ return root
+
+
+def _fresh_build(base_dir, act_int8 = True):
+ model = diffusers.FluxTransformer2DModel.from_pretrained(
+ base_dir, subfolder = "transformer", torch_dtype = torch.bfloat16
+ )
+ view = types.SimpleNamespace(transformer = model)
+ assert (
+ tq._quantize_native(
+ view,
+ "int8",
+ family = FAMILY,
+ min_features = tq.DEFAULT_MIN_LINEAR_FEATURES,
+ act_int8 = act_int8,
+ logger = None,
+ )
+ == "int8"
+ )
+ return model
+
+
+def _assert_same(a, b):
+ sa, sb = a.state_dict(), b.state_dict()
+ assert sa.keys() == sb.keys()
+ for k in sa:
+ assert sa[k].dtype == sb[k].dtype and torch.equal(sa[k], sb[k]), k
+
+
+def _write(
+ base_dir,
+ model,
+ fam = None,
+):
+ pending = NC.save_deferred(
+ model, fam or _fam(), base_repo = "black-forest-labs/FLUX.1-schnell", source_base = base_dir
+ )
+ assert isinstance(pending, NC.NativePendingWrite)
+ assert pending in C._BACKGROUND
+ assert pending.start()
+ assert pending.wait(120)
+ assert pending not in C._BACKGROUND
+ return pending.result
+
+
[email protected]("act_int8", [True, False])
+def test_background_write_round_trips_bit_identical(base_dir, act_int8):
+ built = _fresh_build(base_dir, act_int8)
+ assert _write(base_dir, built)
+ spec = NC.native_spec("int8", FAMILY, act_int8 = act_int8)
+ assert NC.build_spec(built, FAMILY) == spec
+ read = NC.read_for_load(
+ _fam(),
+ spec,
+ diffusers.FluxTransformer2DModel,
+ base_repo = "black-forest-labs/FLUX.1-schnell",
+ source_base = base_dir,
+ )
+ assert read is not None
+ assert (
+ read._unsloth_runtime_quant_info["native"]
+ and read._unsloth_runtime_quant_info["cache_path"]
+ )
+ _assert_same(built, read)
+
+
+def test_key_tracks_the_recipe(base_dir, monkeypatch):
+ built = _fresh_build(base_dir, True)
+ assert _write(base_dir, built)
+ lookup = lambda spec: NC.lookup( # noqa: E731
+ _fam(), spec, base_repo = "black-forest-labs/FLUX.1-schnell", source_base = base_dir
+ )
+ assert lookup(NC.native_spec("int8", FAMILY, act_int8 = True))
+ # weight-only, another rotation group, another family's exclusions, another base: all misses
+ assert lookup(NC.native_spec("int8", FAMILY, act_int8 = False)) is None
+ monkeypatch.setenv("UNSLOTH_NATIVE_INT8_ROT", "0")
+ assert lookup(NC.native_spec("int8", FAMILY, act_int8 = True)) is None
+ monkeypatch.delenv("UNSLOTH_NATIVE_INT8_ROT")
+ assert (
+ NC.lookup(
+ _fam(),
+ NC.native_spec("int8", FAMILY, act_int8 = True),
+ base_repo = "black-forest-labs/FLUX.1-dev",
+ source_base = base_dir,
+ )
+ is None
+ )
+ # a torchao lookup never serves the native entry
+ assert C.lookup(_fam(), "int8", base_repo = "black-forest-labs/FLUX.1-schnell") is None
+
+
+def test_discarded_write_writes_nothing(base_dir):
+ built = _fresh_build(base_dir, True)
+ pending = NC.save_deferred(
+ built, _fam(), base_repo = "black-forest-labs/FLUX.1-schnell", source_base = base_dir
+ )
+ pending.discard()
+ assert pending.done() and pending not in C._BACKGROUND
+ assert (
+ NC.lookup(
+ _fam(),
+ NC.native_spec("int8", FAMILY, act_int8 = True),
+ base_repo = "black-forest-labs/FLUX.1-schnell",
+ source_base = base_dir,
+ )
+ is None
+ )
+
+
+def test_finish_background_writes_waits_for_a_native_write(base_dir):
+ built = _fresh_build(base_dir, True)
+ pending = NC.save_deferred(
+ built, _fam(), base_repo = "black-forest-labs/FLUX.1-schnell", source_base = base_dir
+ )
+ assert pending.start()
+ assert C.finish_background_writes(timeout = 120)
+ assert pending.done() and pending.result
+
+
+def test_sync_write_when_background_writes_are_off(base_dir, monkeypatch):
+ monkeypatch.setenv(C.ENV_ASYNC, "0")
+ built = _fresh_build(base_dir, True)
+ out = NC.save_deferred(
+ built, _fam(), base_repo = "black-forest-labs/FLUX.1-schnell", source_base = base_dir
+ )
+ assert isinstance(out, str) and Path(out).is_file()
+
+
+def test_backend_save_routes_native_builds_to_the_native_cache(base_dir):
+ built = _fresh_build(base_dir, True)
+ pipe = types.SimpleNamespace(transformer = built)
+ pending = D.DiffusionBackend._save_pipeline_quant_to_cache(
+ pipe,
+ _fam(),
+ [("transformer", built)],
+ loras = None,
+ base = "black-forest-labs/FLUX.1-schnell",
+ source_base = base_dir,
+ defer = True,
+ native_cache = True,
+ )
+ assert isinstance(pending, NC.NativePendingWrite)
+ pending.start()
+ assert pending.wait(120) and pending.result
+
+
+def test_backend_save_skips_a_native_build_the_load_cannot_read_back(base_dir):
+ # MPS / ROCm builds never hit the read path, so writing them would only cost disk and an unload wait.
+ built = _fresh_build(base_dir, True)
+ pipe = types.SimpleNamespace(transformer = built)
+ pending = D.DiffusionBackend._save_pipeline_quant_to_cache(
+ pipe,
+ _fam(),
+ [("transformer", built)],
+ loras = None,
+ base = "black-forest-labs/FLUX.1-schnell",
+ source_base = base_dir,
+ defer = True,
+ )
+ assert pending is None
+ assert (
+ NC.lookup(
+ _fam(),
+ NC.build_spec(built, _fam().name),
+ base_repo = "black-forest-labs/FLUX.1-schnell",
+ source_base = base_dir,
+ )
+ is None
+ )
+
+
+def test_backend_read_hits_after_a_write_and_misses_before(base_dir):
+ spec_args = dict(base = "black-forest-labs/FLUX.1-schnell", source_base = base_dir)
+ read = lambda target: D.DiffusionBackend._native_quant_from_cache( # noqa: E731
+ _fam(), diffusers.FluxTransformer2DModel, "int8", target, **spec_args
+ )
+ assert read(CUDA) is None
+ built = _fresh_build(base_dir, D.native_int8_act(CUDA))
+ assert _write(base_dir, built)
+ hit = read(CUDA)
+ assert hit is not None
+ _assert_same(built, hit)
+ # CPU / MPS targets never read (their weight-only scale dtype is not part of the key)
+ assert read(types.SimpleNamespace(device = "mps", dtype = torch.bfloat16)) is None
+ assert (
+ read(CUDA) is not None
+ and D.DiffusionBackend._native_quant_from_cache(
+ _fam(), diffusers.FluxTransformer2DModel, None, CUDA, **spec_args
+ )
+ is None
+ )
+
+
+def test_cache_disabled_reads_and_writes_nothing(base_dir, monkeypatch):
+ monkeypatch.setenv(C.ENV_MODE, "0")
+ built = _fresh_build(base_dir, True)
+ assert (
+ NC.save_deferred(
+ built, _fam(), base_repo = "black-forest-labs/FLUX.1-schnell", source_base = base_dir
+ )
+ is None
+ )