Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio Qwen-Image-2.1: reuse an unchanged condition image's VAE latents across renders (#17092)

Daniel Hane1e4cc64f5f3
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 81732b52608..c0d777ee415 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -344,6 +344,7 @@ from . import diffusion_cuda_graph as cuda_graph
from .diffusion_block_graph import compile_below_hooks_enabled, compile_pipe_below_offload_hooks
from . import diffusion_render_thread as render_thread
from .diffusion_postprocess import install as install_device_postprocess
+from .diffusion_reference_cache import install as install_reference_vae_cache
from .diffusion_batched import (
chunk_jobs,
is_oom_error,
@@ -15219,6 +15220,7 @@ class DiffusionBackend:
chunk_kwargs = applied_embeddings.rewrite(chunk_kwargs)
# Per pipe, idempotent: a workflow pipe built by from_pipe has its own image processor.
install_device_postprocess(pipe, logger)
+ install_reference_vae_cache(pipe, logger)
protect_ctx = protect_generation(pipe, denoise_steps, logger = logger)
# Per chunk: a later chunk encodes again after an earlier one decoded.
gen.phase = "encode"
diff --git a/studio/backend/core/inference/diffusion_reference_cache.py b/studio/backend/core/inference/diffusion_reference_cache.py
new file mode 100644
index 00000000000..00cf1fb931e
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_reference_cache.py
@@ -0,0 +1,220 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Qwen-Image-2.1 / Turbo: reuse an unchanged condition image's VAE latents across renders (a posterior-mode encode,
+so the generator never enters). Kill switch UNSLOTH_REFERENCE_VAE_CACHE=0."""
+
+from __future__ import annotations
+
+import hashlib
+import inspect
+import os
+import threading
+import weakref
+from collections import OrderedDict
+from typing import Any, Optional
+
+CACHE_ENV = "UNSLOTH_REFERENCE_VAE_CACHE"
+_MARK = "_unsloth_reference_vae_cache"
+_CLASSES = frozenset({"QwenImage21Pipeline"})
+_MAX_ENTRIES = 16
+_MAX_BYTES = 256 << 20
+# VAE attributes that change what ``encode`` returns for the same pixels.
+_VAE_STATE = (
+ "use_tiling",
+ "use_slicing",
+ "use_framewise_encoding",
+ "tile_sample_min_height",
+ "tile_sample_min_width",
+ "tile_sample_stride_height",
+ "tile_sample_stride_width",
+)
+
+STATS = {"hits": 0, "misses": 0, "bypassed": 0}
+
+
+def disabled() -> bool:
+ return (os.environ.get(CACHE_ENV) or "").strip().lower() in ("0", "off", "false", "no")
+
+
+def _mode_encode(fn: Any) -> bool:
+ try:
+ src = inspect.getsource(inspect.unwrap(fn))
+ except (OSError, TypeError):
+ return False
+ return 'sample_mode="argmax"' in src and '"sample"' not in src and ".sample(" not in src
+
+
+def _overrides(module: Any) -> tuple:
+ own = getattr(module, "__dict__", {})
+ out = []
+ for name, value in own.items():
+ if name in ("encode", "_encode", "forward", "tiled_encode") and callable(value):
+ out.append((name, id(getattr(value, "__func__", value))))
+ elif name.startswith("_unsloth_") and isinstance(
+ value, (bool, int, float, str, type(None))
+ ):
+ out.append((name, value))
+ return tuple(sorted(out, key = lambda item: item[0]))
+
+
+def _weights_revision(vae: Any) -> Optional[tuple]:
+ # Bumped by in-place writes (load_state_dict), not by offload moves that swap ``.data``.
+ try:
+ tensors = [*vae.parameters(), *vae.buffers()]
+ return (len(tensors), sum(t._version for t in tensors))
+ except Exception: # noqa: BLE001 - untracked weights are never cached
+ return None
+
+
+def _autocast_key(device_types: set) -> Optional[tuple]:
+ # Autocast changes the encode's arithmetic and output dtype without touching parameter dtypes.
+ import torch
+
+ out = []
+ for device_type in sorted(device_types):
+ try:
+ enabled = torch.is_autocast_enabled(device_type)
+ dtype = torch.get_autocast_dtype(device_type) if enabled else None
+ except Exception: # noqa: BLE001 - an unreadable autocast state is never cached
+ return None
+ out.append((device_type, enabled, str(dtype)))
+ return tuple(out)
+
+
+def _vae_key(vae: Any, image: Any, latent_channels: Any) -> Optional[tuple]:
+ try:
+ param = next(vae.parameters())
+ except Exception: # noqa: BLE001 - no parameters / an unreadable VAE is never cached
+ return None
+ weights = _weights_revision(vae)
+ autocast = _autocast_key({"cpu", param.device.type, image.device.type})
+ if weights is None or autocast is None:
+ return None
+ config = getattr(vae, "config", None)
+ try:
+ config = repr(sorted(dict(config).items())) if config is not None else None
+ except Exception: # noqa: BLE001
+ return None
+ # Not the device or address: offload hooks move parameters between renders.
+ state = tuple(repr(getattr(vae, name, None)) for name in _VAE_STATE)
+ encoder = getattr(vae, "encoder", None)
+ return (
+ id(vae),
+ str(param.dtype),
+ weights,
+ autocast,
+ config,
+ repr(latent_channels),
+ state,
+ _overrides(vae),
+ _overrides(encoder) if encoder is not None else (),
+ )
+
+
+def _pixels_key(image: Any) -> Optional[tuple]:
+ import torch
+
+ if not isinstance(image, torch.Tensor) or image.is_complex() or image.numel() == 0:
+ return None
+ raw = image.detach().contiguous().view(-1).view(torch.uint8).cpu().numpy()
+ digest = hashlib.blake2b(raw.data, digest_size = 20).hexdigest()
+ return (digest, tuple(image.shape), str(image.dtype), str(image.device))
+
+
+class _Cache:
+ def __init__(self) -> None:
+ self.entries: "OrderedDict[tuple, Any]" = OrderedDict()
+ self.bytes = 0
+ self.vae_ref: Any = None
+ self.lock = threading.Lock()
+
+ def clear(self) -> None:
+ self.entries.clear()
+ self.bytes = 0
+
+ def get(self, key: tuple) -> Any:
+ with self.lock:
+ hit = self.entries.get(key)
+ if hit is not None:
+ self.entries.move_to_end(key)
+ return hit
+
+ def put(self, key: tuple, value: Any) -> None:
+ size = value.numel() * value.element_size()
+ if size > _MAX_BYTES:
+ return
+ with self.lock:
+ old = self.entries.pop(key, None)
+ if old is not None:
+ self.bytes -= old.numel() * old.element_size()
+ self.entries[key] = value
+ self.bytes += size
+ while self.entries and (len(self.entries) > _MAX_ENTRIES or self.bytes > _MAX_BYTES):
+ _, dropped = self.entries.popitem(last = False)
+ self.bytes -= dropped.numel() * dropped.element_size()
+
+
+def install(pipe: Any, logger: Any = None) -> bool:
+ if disabled() or type(pipe).__name__ not in _CLASSES:
+ return False
+ if getattr(pipe, _MARK, None) is not None:
+ return True
+ stock_fn = getattr(type(pipe), "_encode_vae_image", None)
+ if not callable(stock_fn) or not _mode_encode(stock_fn):
+ if logger is not None:
+ logger.info(
+ "diffusion.reference_cache: off (the reference encode is not a posterior mode)"
+ )
+ return False
+ stock = getattr(pipe, "_encode_vae_image")
+ cache = _Cache()
+ pipe_ref = weakref.ref(pipe)
+
+ def _encode_vae_image(image: Any, generator: Any) -> Any:
+ import torch
+
+ owner = pipe_ref()
+ vae = getattr(owner, "vae", None) if owner is not None else None
+ if disabled() or vae is None or torch.is_grad_enabled():
+ STATS["bypassed"] += 1
+ return stock(image, generator)
+ pixels = _pixels_key(image)
+ vae_key = _vae_key(vae, image, getattr(owner, "latent_channels", None)) if pixels else None
+ if vae_key is None or pixels is None:
+ STATS["bypassed"] += 1
+ return stock(image, generator)
+ if cache.vae_ref is None or cache.vae_ref() is not vae:
+ with cache.lock:
+ cache.clear()
+ cache.vae_ref = weakref.ref(vae)
+ key = (pixels, vae_key)
+ hit = cache.get(key)
+ if hit is not None:
+ STATS["hits"] += 1
+ return hit.to(device = image.device, copy = True)
+ STATS["misses"] += 1
+ out = stock(image, generator)
+ if isinstance(out, torch.Tensor):
+ cache.put(key, out.detach().to("cpu", copy = True))
+ return out
+
+ _encode_vae_image.__wrapped__ = stock
+ pipe._encode_vae_image = _encode_vae_image
+ setattr(pipe, _MARK, cache)
+ if logger is not None:
+ logger.info("diffusion.reference_cache: condition-image VAE latents reused across renders")
+ return True
+
+
+def uninstall(pipe: Any) -> None:
+ if getattr(pipe, _MARK, None) is None:
+ return
+ try:
+ del pipe._encode_vae_image
+ except AttributeError:
+ pass
+ try:
+ delattr(pipe, _MARK)
+ except AttributeError:
+ pass
diff --git a/studio/backend/tests/test_diffusion_reference_cache.py b/studio/backend/tests/test_diffusion_reference_cache.py
new file mode 100644
index 00000000000..59618ed6117
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_reference_cache.py
@@ -0,0 +1,200 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Reused condition latents equal a fresh encode; every input that changes them encodes again."""
+
+from __future__ import annotations
+
+import pytest
+
+from core.inference import diffusion_reference_cache as rc
+
+torch = pytest.importorskip("torch")
+diffusers = pytest.importorskip("diffusers")
+q21 = pytest.importorskip("diffusers.pipelines.qwenimage21.pipeline_qwenimage21")
+
+
[email protected](autouse = True)
+def _env(monkeypatch):
+ monkeypatch.delenv(rc.CACHE_ENV, raising = False)
+ for k in rc.STATS:
+ rc.STATS[k] = 0
+
+
+def _vae(seed = 0):
+ torch.manual_seed(seed)
+ vae = diffusers.AutoencoderKLQwenImage21(
+ base_dim = 8,
+ decoder_base_dim = 8,
+ z_dim = 4,
+ dim_mult = [1, 2, 4, 4, 4],
+ num_res_blocks = 1,
+ attn_scales = [],
+ temperal_downsample = [False, True, True, True],
+ in_channels = 4,
+ out_channels = 4,
+ latents_mean = [0.1, -0.2, 0.3, 0.0],
+ latents_std = [1.5, 0.5, 2.0, 1.0],
+ ).eval()
+ return vae
+
+
+def _pipe(vae = None):
+ cls = q21.QwenImage21Pipeline
+ pipe = cls.__new__(cls)
+ pipe.__dict__.update(vae = vae or _vae(), latent_channels = 4, vae_scale_factor = 16)
+ return pipe
+
+
+def _spy(vae):
+ calls = []
+ real = vae.encode
+
+ def encode(*a, **k):
+ calls.append(1)
+ return real(*a, **k)
+
+ vae.__dict__["encode"] = encode
+ return calls
+
+
+def _images(seed = 1, n = 2):
+ g = torch.Generator().manual_seed(seed)
+ return [torch.rand(1, 4, 1, 64, 48, generator = g) * 2 - 1 for _ in range(n)]
+
+
+def _prepare(pipe, images, seed):
+ g = torch.Generator().manual_seed(seed)
+ with torch.inference_mode():
+ lat, img = pipe.prepare_latents(images, 1, 4, 64, 64, torch.float32, torch.device("cpu"), g)
+ return lat, img, g.get_state()
+
+
+def test_reuse_is_exact_and_skips_the_encode():
+ images = _images()
+ stock = _pipe()
+ ref = [_prepare(stock, images, s) for s in (3, 4, 3)]
+ pipe = _pipe()
+ calls = _spy(pipe.vae)
+ assert rc.install(pipe) and rc.install(pipe)
+ got = [_prepare(pipe, images, s) for s in (3, 4, 3)]
+ for (rl, ri, rg), (gl, gi, gg) in zip(ref, got):
+ assert torch.equal(ri, gi) and torch.equal(rl, gl) and torch.equal(rg, gg)
+ assert len(calls) == len(images)
+ assert rc.STATS["misses"] == 2 and rc.STATS["hits"] == 4
+ rc.uninstall(pipe)
+ _prepare(pipe, images, 3)
+ assert len(calls) == 2 * len(images)
+
+
+def test_every_changed_input_encodes_again():
+ images = _images(n = 1)
+ pipe = _pipe()
+ calls = _spy(pipe.vae)
+ rc.install(pipe)
+ _prepare(pipe, images, 0)
+ other = [images[0].clone()]
+ other[0][0, 0, 0, 0, 0] += 0.25
+ _prepare(pipe, other, 0)
+ assert len(calls) == 2
+ pipe.vae.use_tiling = True
+ _, tiled, _ = _prepare(pipe, images, 0) # tiling changes what encode returns
+ assert len(calls) == 3
+ pipe.vae.use_tiling = False
+ _prepare(pipe, images, 0)
+ assert len(calls) == 3
+ pipe.vae.to(torch.float64)
+ with torch.inference_mode():
+ pipe.prepare_latents(images, 1, 4, 64, 64, torch.float64, torch.device("cpu"), None)
+ assert len(calls) == 4 # a cast VAE (and float64 pixels)
+ pipe.vae.to(torch.float32)
+ fresh = _vae(seed = 5)
+ fresh_calls = _spy(fresh)
+ pipe.__dict__["vae"] = fresh
+ _, img, _ = _prepare(pipe, images, 0)
+ assert len(fresh_calls) == 1 # another VAE object: the cache starts over
+ _, ref, _ = _prepare(_pipe(_vae(seed = 5)), images, 0)
+ assert torch.equal(img, ref)
+
+
+def test_kill_switch(monkeypatch):
+ pipe = _pipe()
+ calls = _spy(pipe.vae)
+ monkeypatch.setenv(rc.CACHE_ENV, "0")
+ assert not rc.install(pipe)
+ monkeypatch.delenv(rc.CACHE_ENV)
+ assert rc.install(pipe)
+ monkeypatch.setenv(rc.CACHE_ENV, "0")
+ _prepare(pipe, _images(n = 1), 0)
+ _prepare(pipe, _images(n = 1), 0)
+ assert len(calls) == 2 and rc.STATS["bypassed"] == 2
+
+
+def test_a_sampled_encode_is_never_cached(monkeypatch):
+ def sampled(self, image, generator):
+ return q21.retrieve_latents(
+ self.vae.encode(image), generator = generator, sample_mode = "sample"
+ )
+
+ monkeypatch.setattr(q21.QwenImage21Pipeline, "_encode_vae_image", sampled)
+ assert not rc.install(_pipe())
+
+
+def test_other_pipelines_are_left_alone():
+ class Other:
+ def _encode_vae_image(self, image, generator):
+ return image
+
+ assert not rc.install(Other())
+
+
+def test_the_cache_is_bounded(monkeypatch):
+ monkeypatch.setattr(rc, "_MAX_ENTRIES", 2)
+ pipe = _pipe()
+ calls = _spy(pipe.vae)
+ rc.install(pipe)
+ sets = [_images(seed = s, n = 1) for s in range(3)]
+ for images in sets:
+ _prepare(pipe, images, 0)
+ _prepare(pipe, sets[0], 0) # LRU-evicted
+ _prepare(pipe, sets[2], 0)
+ assert len(calls) == 4
+ assert len(getattr(pipe, rc._MARK).entries) == 2
+
+
+def _encode(
+ pipe,
+ image,
+ autocast = False,
+):
+ with torch.inference_mode(), torch.autocast("cpu", dtype = torch.bfloat16, enabled = autocast):
+ return pipe._encode_vae_image(image, None), pipe._encode_vae_image.__wrapped__(image, None)
+
+
+def test_autocast_is_part_of_the_key():
+ pipe = _pipe()
+ image = _images(n = 1)[0]
+ rc.install(pipe)
+ _encode(pipe, image)
+ got, want = _encode(pipe, image, autocast = True)
+ assert got.dtype == want.dtype == torch.bfloat16 and torch.equal(got, want)
+ got, want = _encode(pipe, image)
+ assert got.dtype == torch.float32 and torch.equal(got, want)
+ assert rc.STATS["misses"] == 2 and rc.STATS["hits"] == 1
+
+
+def test_reloaded_weights_and_config_encode_again():
+ pipe = _pipe()
+ image = _images(n = 1)[0]
+ rc.install(pipe)
+ _encode(pipe, image)
+ pipe.vae.to(torch.float64).to(torch.float32) # an offload-style move keeps the entry
+ _encode(pipe, image)
+ assert rc.STATS["hits"] == 1
+ pipe.vae.load_state_dict(_vae(seed = 9).state_dict()) # same object, new weights
+ got, want = _encode(pipe, image)
+ assert torch.equal(got, want)
+ pipe.vae.register_to_config(latents_mean = [0.5, 0.5, 0.5, 0.5])
+ got, want = _encode(pipe, image)
+ assert torch.equal(got, want)
+ assert rc.STATS["misses"] == 3 and rc.STATS["hits"] == 1