Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: cache image native int8 builds in the local prequant cache (#15283)
Daniel Han48a9c8dd7090
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex eda31a3b873..e9f4d360a83 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -8707,6 +8707,12 @@ class DiffusionBackend:native_offload_forced = native_offload_scheme is not None and (_memory_request_forces_offload(memory_mode, cpu_offload))+ # The only build the native cache can serve back (MPS / ROCm set native_scheme up front).+ native_cache_ok = (+ native_offload_forced+ and native_scheme is None+ and getattr(target, "device", None) == "cuda"+ )if transformer_quant_pinned is not None and not dense_quant_supported_kind(kind):transformer_quant_decline = dense_quant_unsupported_kind_reason(kind)transformer_quant_decline_status = RESOLVED_UNSUPPORTED@@ -9683,6 +9689,34 @@ class DiffusionBackend:family = getattr(fam, "name", None),logger = logger,)+ if (+ pipeline_seed_scheme is None+ and native_cache_ok+ and not _has_active_lora(loras)+ and not _pipeline_prequant_skipped+ and "transformer" not in pipe_kwargs+ ):+ native_cached = self._native_quant_from_cache(+ fam,+ transformer_cls,+ native_offload_scheme,+ target,+ base = base,+ source_base = _base_local_dir or fetch_base,+ config_kwargs = {+ "local_files_only": local_files_only,+ "cache_dir": hub_cache_dir(),+ **({"token": hf_token} if hf_token else {}),+ },+ cancel_check = lambda: self._raise_if_load_cancelled(+ _load_token+ ),+ )+ if native_cached is not None:+ # Cached native build: skips loading dense shards and re-quantizing.+ pipe_kwargs["transformer"] = native_cached+ native_scheme = native_offload_scheme+ transformer_quant_engaged = native_offload_schemeif pipeline_seed_scheme is None and _pipeline_prequant_skipped:# from_pretrained cannot re-fetch these from a local snapshot dir, so top them up here._base_local_dir = self._restore_skipped_transformer_shards(@@ -10315,6 +10349,7 @@ class DiffusionBackend:_load_token),defer = True,+ native_cache = native_cache_ok,)if not hasattr(pending_cache_write, "start"):pending_cache_write = None@@ -11782,6 +11817,50 @@ class DiffusionBackend:logger.warning("diffusion.prequant_cache: read skipped (%s)", exc)return None+ @staticmethod+ def _native_quant_from_cache(+ fam: Any,+ transformer_cls: Any,+ scheme: Optional[str],+ target: Any,+ *,+ base: Optional[str],+ source_base: Optional[str],+ config_kwargs: Optional[dict] = None,+ cancel_check: Optional[Callable[[], None]] = None,+ ) -> Any:+ """The forced-offload native ``transformer`` from the local prequant cache, or None. CUDA only. Never raises."""+ try:+ if fam is None or scheme is None or getattr(target, "device", None) != "cuda":+ return None+ from .diffusion_native_cache import native_spec, read_for_load++ spec = native_spec(+ scheme,+ getattr(fam, "name", None),+ act_int8 = scheme == TQ_INT8 and native_int8_act(target),+ )+ module = read_for_load(+ fam,+ spec,+ transformer_cls,+ base_repo = base,+ source_base = source_base,+ config_kwargs = config_kwargs,+ logger = logger,+ cancel_check = cancel_check,+ )+ if module is not None:+ logger.info(+ "diffusion.transformer_quant: native %s read from the local prequant cache "+ "(the dense transformer is not loaded, no re-quantize)",+ scheme,+ )+ return module+ except Exception as exc: # noqa: BLE001 - a cache problem is a miss+ logger.warning("diffusion.prequant_cache: native read skipped (%s)", exc)+ return None+@staticmethoddef _save_pipeline_quant_to_cache(pipe: Any,@@ -11793,13 +11872,45 @@ class DiffusionBackend:source_base: Optional[str],cancel_check: Optional[Callable[[], None]] = None,defer: bool = False,+ native_cache: bool = False,) -> Any:"""Cache a load-time quantized ``transformer`` (single denoiser only, the cache keys that component; no LoRA).- ``defer`` may return a ``PendingWrite`` for the caller to start once the load is published. Never raises."""+ ``defer`` may return a ``PendingWrite`` for the caller to start once the load is published. A native build is+ written only when ``native_cache`` says the load could read it back. Never raises."""try:names = [attr for attr, _module in (denoisers or ())]if fam is None or names != ["transformer"] or _has_active_lora(loras):return None+ transformer = getattr(pipe, "transformer", None)+ if (getattr(transformer, "_unsloth_runtime_quant_info", None) or {}).get("native"):+ if not native_cache:+ return None+ from .diffusion_native_cache import save_deferred++ if not defer:+ from .diffusion_native_cache import build_spec, save+ spec = build_spec(transformer, getattr(fam, "name", None))+ return (+ None+ if spec is None+ else save(+ transformer,+ fam,+ spec,+ base_repo = base,+ source_base = source_base,+ logger = logger,+ cancel_check = cancel_check,+ )+ )+ return save_deferred(+ transformer,+ fam,+ base_repo = base,+ source_base = source_base,+ logger = logger,+ cancel_check = cancel_check,+ )from .diffusion_prequant_cache import save_runtime_quantreturn save_runtime_quant(diff --git a/studio/backend/core/inference/diffusion_native_cache.py b/studio/backend/core/inference/diffusion_native_cache.pyindex 454f3871f5f..3f4b1420461 100644--- a/studio/backend/core/inference/diffusion_native_cache.py+++ b/studio/backend/core/inference/diffusion_native_cache.py@@ -454,3 +454,183 @@ def load(except Exception: # noqa: BLE001passreturn None+++def build_spec(transformer: Any, family: Optional[str]) -> Optional[dict]:+ """``native_spec`` of an in-memory native build, read from its layers, or None (no native layers, or a mix)."""+ try:+ from .diffusion_native_quant import is_native_linear++ schemes, acts = set(), set()+ for mod in transformer.modules():+ if is_native_linear(mod):+ schemes.add(mod.scheme)+ acts.add(bool(mod.act_int8))+ if len(schemes) != 1 or len(acts) != 1:+ return None+ return native_spec(next(iter(schemes)), family, act_int8 = next(iter(acts)))+ except Exception: # noqa: BLE001+ return None+++class NativePendingWrite:+ """Background ``save``, awaited by ``finish_background_writes``; writes live host tensors (no snapshot copy)."""++ def __init__(+ self,+ run: Any,+ key: str,+ logger: Any = None,+ ) -> None:+ import threading++ self._run_fn = run+ self.key = key+ self._logger = logger+ self._cancel = threading.Event()+ self._finished = threading.Event()+ self._thread: Optional[Any] = None+ self.result: Optional[str] = None+ with _cache._lock:+ _cache._BACKGROUND.add(self)++ def _check(self) -> None:+ if self._cancel.is_set():+ raise RuntimeError("native cache write cancelled")++ def start(self) -> bool:+ import threading++ with _cache._lock:+ if self._thread is not None or self._run_fn is None or self._cancel.is_set():+ return False+ self._thread = threading.Thread(+ target = self._run, name = f"native-cache-write-{self.key[:8]}", daemon = True+ )+ try:+ self._thread.start()+ except Exception: # noqa: BLE001 - no writer: the next load re-quantizes+ with _cache._lock:+ self._thread = None+ self._close()+ return False+ return True++ def _run(self) -> None:+ try:+ if self._run_fn is not None:+ self.result = self._run_fn(self._check)+ except Exception as exc: # noqa: BLE001 - a background cache write never surfaces+ _cache._log(self._logger, "warning", f"native: background save failed: {exc}")+ finally:+ self._close()++ def _close(self) -> None:+ self._run_fn = None+ with _cache._lock:+ _cache._BACKGROUND.discard(self)+ self._finished.set()++ def discard(self) -> None:+ with _cache._lock:+ started = self._thread is not None+ self._cancel.set()+ if not started:+ self._close()++ def cancel(self) -> None:+ self.discard()++ def done(self) -> bool:+ return self._finished.is_set()++ def wait(self, timeout: Optional[float] = None) -> bool:+ return self._finished.wait(timeout)+++def save_deferred(+ transformer: Any,+ fam: Any,+ *,+ base_repo: Optional[str],+ source_base: Optional[str] = None,+ component: str = "transformer",+ logger: Any = None,+ cancel_check: Optional[Any] = None,+) -> Any:+ """A ``NativePendingWrite`` (start after publish), the sync ``save`` result, or None. Never raises."""+ try:+ if not _cache.enabled():+ return None+ spec = build_spec(transformer, getattr(fam, "name", None))+ if spec is None:+ _cache._log(logger, "info", f"native {component}: skipped (no single native recipe)")+ return None+ recorded_base = str(base_repo or getattr(fam, "base_repo", "") or "")+ keyed = _key_for(fam, spec, source_base or base_repo, component, base_id = recorded_base)+ if keyed is None:+ _cache._log(+ logger,+ "info",+ f"native {component}: skipped (dense source shards not found locally)",+ )+ return None++ def run(check: Optional[Any] = None) -> Optional[str]:+ def both() -> None:+ if check is not None:+ check()+ if cancel_check is not None and check is None:+ cancel_check()++ return save(+ transformer,+ fam,+ spec,+ base_repo = base_repo,+ source_base = source_base,+ component = component,+ logger = logger,+ cancel_check = both,+ )++ if not _cache.async_enabled():+ return run()+ return NativePendingWrite(run, keyed[0], logger)+ except Exception as exc: # noqa: BLE001+ _cache._log(+ logger, "warning", f"native {component}: save skipped ({type(exc).__name__}: {exc})"+ )+ return None+++def read_for_load(+ fam: Any,+ spec: Optional[dict],+ transformer_cls: Any,+ *,+ base_repo: Optional[str],+ source_base: Optional[str],+ component: str = "transformer",+ config_kwargs: Optional[dict] = None,+ logger: Any = None,+ cancel_check: Optional[Any] = None,+) -> Any:+ """``lookup`` + ``load`` of the cached native ``component``, or None (a miss). Never raises."""+ if spec is None or transformer_cls is None:+ return None+ path = lookup(+ fam, spec, base_repo = base_repo, source_base = source_base, component = component, logger = logger+ )+ if path is None:+ return None+ return load(+ path,+ transformer_cls,+ str(source_base or base_repo),+ component = component,+ spec = spec,+ config_kwargs = config_kwargs,+ logger = logger,+ cancel_check = cancel_check,+ )diff --git a/studio/backend/tests/test_diffusion_image_native_prequant_cache.py b/studio/backend/tests/test_diffusion_image_native_prequant_cache.pynew file mode 100644index 00000000000..e50da16158c--- /dev/null+++ b/studio/backend/tests/test_diffusion_image_native_prequant_cache.py@@ -0,0 +1,265 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Image native int8 builds are cached in the background and read back instead of re-quantizing."""++from __future__ import annotations++import sys+import types+from pathlib import Path++import pytest++torch = pytest.importorskip("torch")+diffusers = pytest.importorskip("diffusers")+pytest.importorskip("safetensors")+pytest.importorskip("accelerate")++BACKEND = Path(__file__).resolve().parents[1]+if str(BACKEND) not in sys.path:+ sys.path.insert(0, str(BACKEND))++from core.inference import diffusion as D # noqa: E402+from core.inference import diffusion_native_cache as NC # noqa: E402+from core.inference import diffusion_prequant_cache as C # noqa: E402+from core.inference import diffusion_transformer_quant as tq # noqa: E402+from core.inference.diffusion_families import detect_family # noqa: E402++FAMILY = "flux.1"+CUDA = types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16)+++def _fam():+ return detect_family("black-forest-labs/FLUX.1-schnell", override = FAMILY)+++def base_dir(tmp_path):+ root = tmp_path / "FLUX.1-schnell"+ torch.manual_seed(0)+ # 2 x 256 = 512 wide: every block Linear passes the 512 filter and g256 ConvRot divides it.+ model = diffusers.FluxTransformer2DModel(+ patch_size = 1,+ in_channels = 16,+ num_layers = 1,+ num_single_layers = 1,+ attention_head_dim = 256,+ num_attention_heads = 2,+ joint_attention_dim = 512,+ pooled_projection_dim = 64,+ axes_dims_rope = (32, 112, 112),+ )+ model.save_pretrained(str(root / "transformer"), safe_serialization = True)+ return str(root)++[email protected](autouse = True)+def cache_root(tmp_path, monkeypatch):+ root = tmp_path / "cache"+ monkeypatch.setenv(C.ENV_DIR, str(root))+ monkeypatch.delenv(C.ENV_MODE, raising = False)+ monkeypatch.delenv(C.ENV_ASYNC, raising = False)+ monkeypatch.setattr(C, "_hub_source_stale", lambda shards, logger = None: False)+ return root+++def _fresh_build(base_dir, act_int8 = True):+ model = diffusers.FluxTransformer2DModel.from_pretrained(+ base_dir, subfolder = "transformer", torch_dtype = torch.bfloat16+ )+ view = types.SimpleNamespace(transformer = model)+ assert (+ tq._quantize_native(+ view,+ "int8",+ family = FAMILY,+ min_features = tq.DEFAULT_MIN_LINEAR_FEATURES,+ act_int8 = act_int8,+ logger = None,+ )+ == "int8"+ )+ return model+++def _assert_same(a, b):+ sa, sb = a.state_dict(), b.state_dict()+ assert sa.keys() == sb.keys()+ for k in sa:+ assert sa[k].dtype == sb[k].dtype and torch.equal(sa[k], sb[k]), k+++def _write(+ base_dir,+ model,+ fam = None,+):+ pending = NC.save_deferred(+ model, fam or _fam(), base_repo = "black-forest-labs/FLUX.1-schnell", source_base = base_dir+ )+ assert isinstance(pending, NC.NativePendingWrite)+ assert pending in C._BACKGROUND+ assert pending.start()+ assert pending.wait(120)+ assert pending not in C._BACKGROUND+ return pending.result++[email protected]("act_int8", [True, False])+def test_background_write_round_trips_bit_identical(base_dir, act_int8):+ built = _fresh_build(base_dir, act_int8)+ assert _write(base_dir, built)+ spec = NC.native_spec("int8", FAMILY, act_int8 = act_int8)+ assert NC.build_spec(built, FAMILY) == spec+ read = NC.read_for_load(+ _fam(),+ spec,+ diffusers.FluxTransformer2DModel,+ base_repo = "black-forest-labs/FLUX.1-schnell",+ source_base = base_dir,+ )+ assert read is not None+ assert (+ read._unsloth_runtime_quant_info["native"]+ and read._unsloth_runtime_quant_info["cache_path"]+ )+ _assert_same(built, read)+++def test_key_tracks_the_recipe(base_dir, monkeypatch):+ built = _fresh_build(base_dir, True)+ assert _write(base_dir, built)+ lookup = lambda spec: NC.lookup( # noqa: E731+ _fam(), spec, base_repo = "black-forest-labs/FLUX.1-schnell", source_base = base_dir+ )+ assert lookup(NC.native_spec("int8", FAMILY, act_int8 = True))+ # weight-only, another rotation group, another family's exclusions, another base: all misses+ assert lookup(NC.native_spec("int8", FAMILY, act_int8 = False)) is None+ monkeypatch.setenv("UNSLOTH_NATIVE_INT8_ROT", "0")+ assert lookup(NC.native_spec("int8", FAMILY, act_int8 = True)) is None+ monkeypatch.delenv("UNSLOTH_NATIVE_INT8_ROT")+ assert (+ NC.lookup(+ _fam(),+ NC.native_spec("int8", FAMILY, act_int8 = True),+ base_repo = "black-forest-labs/FLUX.1-dev",+ source_base = base_dir,+ )+ is None+ )+ # a torchao lookup never serves the native entry+ assert C.lookup(_fam(), "int8", base_repo = "black-forest-labs/FLUX.1-schnell") is None+++def test_discarded_write_writes_nothing(base_dir):+ built = _fresh_build(base_dir, True)+ pending = NC.save_deferred(+ built, _fam(), base_repo = "black-forest-labs/FLUX.1-schnell", source_base = base_dir+ )+ pending.discard()+ assert pending.done() and pending not in C._BACKGROUND+ assert (+ NC.lookup(+ _fam(),+ NC.native_spec("int8", FAMILY, act_int8 = True),+ base_repo = "black-forest-labs/FLUX.1-schnell",+ source_base = base_dir,+ )+ is None+ )+++def test_finish_background_writes_waits_for_a_native_write(base_dir):+ built = _fresh_build(base_dir, True)+ pending = NC.save_deferred(+ built, _fam(), base_repo = "black-forest-labs/FLUX.1-schnell", source_base = base_dir+ )+ assert pending.start()+ assert C.finish_background_writes(timeout = 120)+ assert pending.done() and pending.result+++def test_sync_write_when_background_writes_are_off(base_dir, monkeypatch):+ monkeypatch.setenv(C.ENV_ASYNC, "0")+ built = _fresh_build(base_dir, True)+ out = NC.save_deferred(+ built, _fam(), base_repo = "black-forest-labs/FLUX.1-schnell", source_base = base_dir+ )+ assert isinstance(out, str) and Path(out).is_file()+++def test_backend_save_routes_native_builds_to_the_native_cache(base_dir):+ built = _fresh_build(base_dir, True)+ pipe = types.SimpleNamespace(transformer = built)+ pending = D.DiffusionBackend._save_pipeline_quant_to_cache(+ pipe,+ _fam(),+ [("transformer", built)],+ loras = None,+ base = "black-forest-labs/FLUX.1-schnell",+ source_base = base_dir,+ defer = True,+ native_cache = True,+ )+ assert isinstance(pending, NC.NativePendingWrite)+ pending.start()+ assert pending.wait(120) and pending.result+++def test_backend_save_skips_a_native_build_the_load_cannot_read_back(base_dir):+ # MPS / ROCm builds never hit the read path, so writing them would only cost disk and an unload wait.+ built = _fresh_build(base_dir, True)+ pipe = types.SimpleNamespace(transformer = built)+ pending = D.DiffusionBackend._save_pipeline_quant_to_cache(+ pipe,+ _fam(),+ [("transformer", built)],+ loras = None,+ base = "black-forest-labs/FLUX.1-schnell",+ source_base = base_dir,+ defer = True,+ )+ assert pending is None+ assert (+ NC.lookup(+ _fam(),+ NC.build_spec(built, _fam().name),+ base_repo = "black-forest-labs/FLUX.1-schnell",+ source_base = base_dir,+ )+ is None+ )+++def test_backend_read_hits_after_a_write_and_misses_before(base_dir):+ spec_args = dict(base = "black-forest-labs/FLUX.1-schnell", source_base = base_dir)+ read = lambda target: D.DiffusionBackend._native_quant_from_cache( # noqa: E731+ _fam(), diffusers.FluxTransformer2DModel, "int8", target, **spec_args+ )+ assert read(CUDA) is None+ built = _fresh_build(base_dir, D.native_int8_act(CUDA))+ assert _write(base_dir, built)+ hit = read(CUDA)+ assert hit is not None+ _assert_same(built, hit)+ # CPU / MPS targets never read (their weight-only scale dtype is not part of the key)+ assert read(types.SimpleNamespace(device = "mps", dtype = torch.bfloat16)) is None+ assert (+ read(CUDA) is not None+ and D.DiffusionBackend._native_quant_from_cache(+ _fam(), diffusers.FluxTransformer2DModel, None, CUDA, **spec_args+ )+ is None+ )+++def test_cache_disabled_reads_and_writes_nothing(base_dir, monkeypatch):+ monkeypatch.setenv(C.ENV_MODE, "0")+ built = _fresh_build(base_dir, True)+ assert (+ NC.save_deferred(+ built, _fam(), base_repo = "black-forest-labs/FLUX.1-schnell", source_base = base_dir+ )+ is None+ )