Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio Sol-H3 VAE: decide window masks from the geometry, build each RoPE table once per decode (#17100)
Daniel Hanac8be67fd07d
diff --git a/studio/backend/core/inference/video_minimax_h3_sol_vae.py b/studio/backend/core/inference/video_minimax_h3_sol_vae.pyindex e6e34405cde..d529482ed2d 100644--- a/studio/backend/core/inference/video_minimax_h3_sol_vae.py+++ b/studio/backend/core/inference/video_minimax_h3_sol_vae.py@@ -24,6 +24,7 @@ import timefrom typing import Any, OptionalSOL_VAE_ENV = "UNSLOTH_H3_SOL_VAE"+SOL_LEAN_WINDOWS_ENV = "UNSLOTH_H3_SOL_VAE_LEAN_WINDOWS"SOL_VAE_REPO = "Efficient-Large-Model/Sol-H3-VAE"SOL_VAE_FILE = "sol_h3_vae.safetensors"SOL_LAYER = "h3_sol_vae"@@ -50,6 +51,15 @@ def h3_sol_vae_requested() -> bool:return os.environ.get(SOL_VAE_ENV, "").strip().lower() in _TRUE+def _lean_windows() -> bool:+ return os.environ.get(SOL_LEAN_WINDOWS_ENV, "").strip().lower() not in (+ "0",+ "false",+ "no",+ "off",+ )++def sol_num_frames(latents: int) -> int:"""Frames Sol (and the stock H3 decoder) returns for ``latents`` latent frames: 4t - 3*ceil(t/5)."""t = int(latents)@@ -233,13 +243,26 @@ def _build_module_classes():cdt = self.compute_dtypeh = fused._add_rmsnorm(x[t0:t1], None, None, blk.norm1, cdt).reshape(-1, D)q, k, v = blk.to_q(h), blk.to_k(h), blk.to_v(h)- cos, sin = self._rope(t0, t1 - t0, H, W, s, st, x.device)- cos, sin = cos.to(cdt).contiguous(), sin.to(cdt).contiguous()+ cos, sin = self._slot_rope(t0, t1 - t0, H, W, s, st, x.device)fused._qk_norm_rope_(q, cos, sin, self.heads, self.eps)fused._qk_norm_rope_(k, cos, sin, self.heads, self.eps)n = t1 - t0return q.view(n, H, W, D), k.view(n, H, W, D), v.view(n, H, W, D)+ def _slot_rope(+ self, t0: int, n: int, H: int, W: int, s: int, st: int, device: Any+ ) -> tuple:+ """``_rope`` in the compute dtype, cached per decode: with t0 a multiple of block it does not depend on t0."""+ cache = getattr(self, "_rope_tables", None)+ key = (n, H, W, s, st, str(device))+ if cache is not None and key in cache:+ return cache[key]+ cos, sin = self._rope(t0, n, H, W, s, st, device)+ hit = (cos.to(self.compute_dtype).contiguous(), sin.to(self.compute_dtype).contiguous())+ if cache is not None:+ cache[key] = hit+ return hit+def _global_attn(self, qg: Any, kg: Any, vg: Any, k: Any, v: Any) -> Any:"""A block's S global tokens over its own tokens + its global tokens: (S, D)."""S, D = qg.shape@@ -255,7 +278,8 @@ def _build_module_classes():self, q: Any, k: Any, v: Any, kgw: Any, vgw: Any, gvalid: Any, k0: int, s: int) -> Any:"""Attention over every spatial window of one temporal window. q/k/v (n, H, W, D) start ``k0`` latents into- the padded window; kgw / vgw (G, D) are the global tokens of the blocks it overlaps, gvalid (G,) bool."""+ the padded window; kgw / vgw (G, D) are the global tokens of the blocks it overlaps, gvalid (G,) bool or+ None when all are valid."""n, H, W, D = q.shapews, tb, hd, dh = self.window, self.block, self.heads, self.dhHp = -(-(H + s) // ws) * ws@@ -269,20 +293,26 @@ def _build_module_classes():buf = buf.view(tb, nh, ws, nw, ws, hd, dh).permute(1, 3, 5, 0, 2, 4, 6)return buf.reshape(nW, hd, L, dh)- valid = torch.zeros(tb, Hp, Wp, dtype = torch.bool, device = q.device)- valid[k0 : k0 + n, s : s + H, s : s + W] = True- valid = valid.view(tb, nh, ws, nw, ws).permute(1, 3, 0, 2, 4).reshape(nW, L)G = kgw.shape[0]kg = kgw.view(G, hd, dh).transpose(0, 1)[None].expand(nW, hd, G, dh)vg = vgw.view(G, hd, dh).transpose(0, 1)[None].expand(nW, hd, G, dh)keys = torch.cat([part(k), kg], 2)vals = torch.cat([part(v), vg], 2)- kmask = torch.cat([valid, gvalid[None].expand(nW, G)], 1)mask = None- if not bool(kmask.all()):- mask = torch.zeros(kmask.shape, dtype = q.dtype, device = q.device)- mask.masked_fill_(~kmask, float("-inf"))- mask = mask[:, None, None, :]+ # every padded slot holds a token iff the window starts at 0, is full, unshifted and unpadded+ lean = gvalid is None and _lean_windows()+ full = k0 == 0 and n == tb and s == 0 and Hp == H and Wp == W+ if not (lean and full):+ valid = torch.zeros(tb, Hp, Wp, dtype = torch.bool, device = q.device)+ valid[k0 : k0 + n, s : s + H, s : s + W] = True+ valid = valid.view(tb, nh, ws, nw, ws).permute(1, 3, 0, 2, 4).reshape(nW, L)+ if gvalid is None:+ gvalid = torch.ones(G, dtype = torch.bool, device = q.device)+ kmask = torch.cat([valid, gvalid[None].expand(nW, G)], 1)+ if lean or not bool(kmask.all()):+ mask = torch.zeros(kmask.shape, dtype = q.dtype, device = q.device)+ mask.masked_fill_(~kmask, float("-inf"))+ mask = mask[:, None, None, :]out = torch.nn.functional.scaled_dot_product_attention(part(q), keys, vals, attn_mask = mask)@@ -336,12 +366,10 @@ def _build_module_classes():)if b1 != b0:kgw, vgw = kg[[b0, b1]].reshape(-1, D), vg[[b0, b1]].reshape(-1, D)- gvalid = torch.ones(2 * S, dtype = torch.bool, device = x.device)else:# the reference pads the second block slot and masks it; leaving it out is the same softmaxkgw, vgw = kg[b0], vg[b0]- gvalid = torch.ones(S, dtype = torch.bool, device = x.device)- o = self._window_attn(q, k, v, kgw, vgw, gvalid, lo - (j * tb - st), s)+ o = self._window_attn(q, k, v, kgw, vgw, None, lo - (j * tb - st), s)del q, k, v, partsxw = x[lo : hi + 1].view(-1, D)normed = fused._add_rmsnorm(@@ -380,8 +408,12 @@ def _build_module_classes():return self._decode_one(z.expand(C, self.block, H, W))[:, :1]x = self._embed(z)g = self._globals(-(-T // self.block), x.device)- for li, blk in enumerate(self.blocks):- self._layer_(li, blk, x, g)+ self._rope_tables = {} if _lean_windows() else None+ try:+ for li, blk in enumerate(self.blocks):+ self._layer_(li, blk, x, g)+ finally:+ self._rope_tables = Noneout = torch.empty(OUT_CH,sol_num_frames(T),diff --git a/studio/backend/tests/test_video_minimax_h3_sol_vae.py b/studio/backend/tests/test_video_minimax_h3_sol_vae.pyindex b515c2163c9..b0faab3ef04 100644--- a/studio/backend/tests/test_video_minimax_h3_sol_vae.py+++ b/studio/backend/tests/test_video_minimax_h3_sol_vae.py@@ -432,3 +432,53 @@ def test_the_h3_load_installs_sol_after_the_stock_levers_and_under_the_oom_scope}# the audio VAE is never handed to itassert "audio_vae" not in ast.unparse(install)+++# ------------------------------------------------------------------------------------------------ host reads+def _count_host_reads(monkeypatch) -> dict:+ n = {"reads": 0}+ orig_bool, orig_item = torch.Tensor.__bool__, torch.Tensor.item++ def counted_bool(self):+ n["reads"] += 1+ return orig_bool(self)++ def counted_item(self):+ n["reads"] += 1+ return orig_item(self)++ monkeypatch.setattr(torch.Tensor, "__bool__", counted_bool)+ monkeypatch.setattr(torch.Tensor, "item", counted_item)+ return n++[email protected]("T,H,W", [(7, 6, 9), (10, 8, 8), (1, 8, 8)])+def test_the_decode_reads_nothing_back_and_matches_the_old_path(monkeypatch, T, H, W):+ """Window masks are decided from the window geometry (host ints), not by reading the mask back per window; the+ kill switch restores the per-window read and must give the same bits."""+ model = _tiny()+ z = torch.randn(1, 24, T, H, W, generator = torch.Generator().manual_seed(T + H))+ monkeypatch.setenv(S.SOL_LEAN_WINDOWS_ENV, "0")+ old = model.decode_raw(z)+ monkeypatch.delenv(S.SOL_LEAN_WINDOWS_ENV)+ n = _count_host_reads(monkeypatch)+ got = model.decode_raw(z)+ assert n["reads"] == 0+ assert torch.equal(got, old)+++def test_the_rope_tables_are_built_once_per_window_geometry():+ model = _tiny()+ calls: list = []+ orig = model._rope++ def rope(t0, n, H, W, s, st, device):+ calls.append((n, s, st))+ return orig(t0, n, H, W, s, st, device)++ model._rope = rope+ model.decode_raw(torch.randn(1, 24, 12, 8, 8, generator = torch.Generator().manual_seed(3)))+ assert len(calls) == len(+ set(calls)+ ) # 3 layers, 3 blocks each: one table per (rows, shift) only+ assert model._rope_tables is None # released with the decode