Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio Sol-H3 VAE: decide window masks from the geometry, build each RoPE table once per decode (#17100)

Daniel Hanac8be67fd07d
diff --git a/studio/backend/core/inference/video_minimax_h3_sol_vae.py b/studio/backend/core/inference/video_minimax_h3_sol_vae.py
index e6e34405cde..d529482ed2d 100644
--- a/studio/backend/core/inference/video_minimax_h3_sol_vae.py
+++ b/studio/backend/core/inference/video_minimax_h3_sol_vae.py
@@ -24,6 +24,7 @@ import time
from typing import Any, Optional
SOL_VAE_ENV = "UNSLOTH_H3_SOL_VAE"
+SOL_LEAN_WINDOWS_ENV = "UNSLOTH_H3_SOL_VAE_LEAN_WINDOWS"
SOL_VAE_REPO = "Efficient-Large-Model/Sol-H3-VAE"
SOL_VAE_FILE = "sol_h3_vae.safetensors"
SOL_LAYER = "h3_sol_vae"
@@ -50,6 +51,15 @@ def h3_sol_vae_requested() -> bool:
return os.environ.get(SOL_VAE_ENV, "").strip().lower() in _TRUE
+def _lean_windows() -> bool:
+ return os.environ.get(SOL_LEAN_WINDOWS_ENV, "").strip().lower() not in (
+ "0",
+ "false",
+ "no",
+ "off",
+ )
+
+
def sol_num_frames(latents: int) -> int:
"""Frames Sol (and the stock H3 decoder) returns for ``latents`` latent frames: 4t - 3*ceil(t/5)."""
t = int(latents)
@@ -233,13 +243,26 @@ def _build_module_classes():
cdt = self.compute_dtype
h = fused._add_rmsnorm(x[t0:t1], None, None, blk.norm1, cdt).reshape(-1, D)
q, k, v = blk.to_q(h), blk.to_k(h), blk.to_v(h)
- cos, sin = self._rope(t0, t1 - t0, H, W, s, st, x.device)
- cos, sin = cos.to(cdt).contiguous(), sin.to(cdt).contiguous()
+ cos, sin = self._slot_rope(t0, t1 - t0, H, W, s, st, x.device)
fused._qk_norm_rope_(q, cos, sin, self.heads, self.eps)
fused._qk_norm_rope_(k, cos, sin, self.heads, self.eps)
n = t1 - t0
return q.view(n, H, W, D), k.view(n, H, W, D), v.view(n, H, W, D)
+ def _slot_rope(
+ self, t0: int, n: int, H: int, W: int, s: int, st: int, device: Any
+ ) -> tuple:
+ """``_rope`` in the compute dtype, cached per decode: with t0 a multiple of block it does not depend on t0."""
+ cache = getattr(self, "_rope_tables", None)
+ key = (n, H, W, s, st, str(device))
+ if cache is not None and key in cache:
+ return cache[key]
+ cos, sin = self._rope(t0, n, H, W, s, st, device)
+ hit = (cos.to(self.compute_dtype).contiguous(), sin.to(self.compute_dtype).contiguous())
+ if cache is not None:
+ cache[key] = hit
+ return hit
+
def _global_attn(self, qg: Any, kg: Any, vg: Any, k: Any, v: Any) -> Any:
"""A block's S global tokens over its own tokens + its global tokens: (S, D)."""
S, D = qg.shape
@@ -255,7 +278,8 @@ def _build_module_classes():
self, q: Any, k: Any, v: Any, kgw: Any, vgw: Any, gvalid: Any, k0: int, s: int
) -> Any:
"""Attention over every spatial window of one temporal window. q/k/v (n, H, W, D) start ``k0`` latents into
- the padded window; kgw / vgw (G, D) are the global tokens of the blocks it overlaps, gvalid (G,) bool."""
+ the padded window; kgw / vgw (G, D) are the global tokens of the blocks it overlaps, gvalid (G,) bool or
+ None when all are valid."""
n, H, W, D = q.shape
ws, tb, hd, dh = self.window, self.block, self.heads, self.dh
Hp = -(-(H + s) // ws) * ws
@@ -269,20 +293,26 @@ def _build_module_classes():
buf = buf.view(tb, nh, ws, nw, ws, hd, dh).permute(1, 3, 5, 0, 2, 4, 6)
return buf.reshape(nW, hd, L, dh)
- valid = torch.zeros(tb, Hp, Wp, dtype = torch.bool, device = q.device)
- valid[k0 : k0 + n, s : s + H, s : s + W] = True
- valid = valid.view(tb, nh, ws, nw, ws).permute(1, 3, 0, 2, 4).reshape(nW, L)
G = kgw.shape[0]
kg = kgw.view(G, hd, dh).transpose(0, 1)[None].expand(nW, hd, G, dh)
vg = vgw.view(G, hd, dh).transpose(0, 1)[None].expand(nW, hd, G, dh)
keys = torch.cat([part(k), kg], 2)
vals = torch.cat([part(v), vg], 2)
- kmask = torch.cat([valid, gvalid[None].expand(nW, G)], 1)
mask = None
- if not bool(kmask.all()):
- mask = torch.zeros(kmask.shape, dtype = q.dtype, device = q.device)
- mask.masked_fill_(~kmask, float("-inf"))
- mask = mask[:, None, None, :]
+ # every padded slot holds a token iff the window starts at 0, is full, unshifted and unpadded
+ lean = gvalid is None and _lean_windows()
+ full = k0 == 0 and n == tb and s == 0 and Hp == H and Wp == W
+ if not (lean and full):
+ valid = torch.zeros(tb, Hp, Wp, dtype = torch.bool, device = q.device)
+ valid[k0 : k0 + n, s : s + H, s : s + W] = True
+ valid = valid.view(tb, nh, ws, nw, ws).permute(1, 3, 0, 2, 4).reshape(nW, L)
+ if gvalid is None:
+ gvalid = torch.ones(G, dtype = torch.bool, device = q.device)
+ kmask = torch.cat([valid, gvalid[None].expand(nW, G)], 1)
+ if lean or not bool(kmask.all()):
+ mask = torch.zeros(kmask.shape, dtype = q.dtype, device = q.device)
+ mask.masked_fill_(~kmask, float("-inf"))
+ mask = mask[:, None, None, :]
out = torch.nn.functional.scaled_dot_product_attention(
part(q), keys, vals, attn_mask = mask
)
@@ -336,12 +366,10 @@ def _build_module_classes():
)
if b1 != b0:
kgw, vgw = kg[[b0, b1]].reshape(-1, D), vg[[b0, b1]].reshape(-1, D)
- gvalid = torch.ones(2 * S, dtype = torch.bool, device = x.device)
else:
# the reference pads the second block slot and masks it; leaving it out is the same softmax
kgw, vgw = kg[b0], vg[b0]
- gvalid = torch.ones(S, dtype = torch.bool, device = x.device)
- o = self._window_attn(q, k, v, kgw, vgw, gvalid, lo - (j * tb - st), s)
+ o = self._window_attn(q, k, v, kgw, vgw, None, lo - (j * tb - st), s)
del q, k, v, parts
xw = x[lo : hi + 1].view(-1, D)
normed = fused._add_rmsnorm(
@@ -380,8 +408,12 @@ def _build_module_classes():
return self._decode_one(z.expand(C, self.block, H, W))[:, :1]
x = self._embed(z)
g = self._globals(-(-T // self.block), x.device)
- for li, blk in enumerate(self.blocks):
- self._layer_(li, blk, x, g)
+ self._rope_tables = {} if _lean_windows() else None
+ try:
+ for li, blk in enumerate(self.blocks):
+ self._layer_(li, blk, x, g)
+ finally:
+ self._rope_tables = None
out = torch.empty(
OUT_CH,
sol_num_frames(T),
diff --git a/studio/backend/tests/test_video_minimax_h3_sol_vae.py b/studio/backend/tests/test_video_minimax_h3_sol_vae.py
index b515c2163c9..b0faab3ef04 100644
--- a/studio/backend/tests/test_video_minimax_h3_sol_vae.py
+++ b/studio/backend/tests/test_video_minimax_h3_sol_vae.py
@@ -432,3 +432,53 @@ def test_the_h3_load_installs_sol_after_the_stock_levers_and_under_the_oom_scope
}
# the audio VAE is never handed to it
assert "audio_vae" not in ast.unparse(install)
+
+
+# ------------------------------------------------------------------------------------------------ host reads
+def _count_host_reads(monkeypatch) -> dict:
+ n = {"reads": 0}
+ orig_bool, orig_item = torch.Tensor.__bool__, torch.Tensor.item
+
+ def counted_bool(self):
+ n["reads"] += 1
+ return orig_bool(self)
+
+ def counted_item(self):
+ n["reads"] += 1
+ return orig_item(self)
+
+ monkeypatch.setattr(torch.Tensor, "__bool__", counted_bool)
+ monkeypatch.setattr(torch.Tensor, "item", counted_item)
+ return n
+
+
[email protected]("T,H,W", [(7, 6, 9), (10, 8, 8), (1, 8, 8)])
+def test_the_decode_reads_nothing_back_and_matches_the_old_path(monkeypatch, T, H, W):
+ """Window masks are decided from the window geometry (host ints), not by reading the mask back per window; the
+ kill switch restores the per-window read and must give the same bits."""
+ model = _tiny()
+ z = torch.randn(1, 24, T, H, W, generator = torch.Generator().manual_seed(T + H))
+ monkeypatch.setenv(S.SOL_LEAN_WINDOWS_ENV, "0")
+ old = model.decode_raw(z)
+ monkeypatch.delenv(S.SOL_LEAN_WINDOWS_ENV)
+ n = _count_host_reads(monkeypatch)
+ got = model.decode_raw(z)
+ assert n["reads"] == 0
+ assert torch.equal(got, old)
+
+
+def test_the_rope_tables_are_built_once_per_window_geometry():
+ model = _tiny()
+ calls: list = []
+ orig = model._rope
+
+ def rope(t0, n, H, W, s, st, device):
+ calls.append((n, s, st))
+ return orig(t0, n, H, W, s, st, device)
+
+ model._rope = rope
+ model.decode_raw(torch.randn(1, 24, 12, 8, 8, generator = torch.Generator().manual_seed(3)))
+ assert len(calls) == len(
+ set(calls)
+ ) # 3 layers, 3 blocks each: one table per (rows, shift) only
+ assert model._rope_tables is None # released with the decode