Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio video: denormalize decoded frames slice by slice in the uint8 export (#16748)
Daniel Han3fc6b699e65e
diff --git a/studio/backend/core/inference/video_frames.py b/studio/backend/core/inference/video_frames.pyindex 0383365ec76..02f202ad0bf 100644--- a/studio/backend/core/inference/video_frames.py+++ b/studio/backend/core/inference/video_frames.py@@ -16,8 +16,9 @@ _FINITE_SLICE_ELEMS = 16 << 20NONFINITE_CHECK_ENV = "UNSLOTH_VIDEO_NONFINITE_CHECK"-def device_uint8(frames: Any) -> Optional[Any]:- """(F, C, H, W) float frames in [0, 1] -> CPU uint8 (F, H, W, C); None when the fast path does not apply."""+def device_uint8(frames: Any, prepare: Any = None) -> Optional[Any]:+ """(F, C, H, W) float frames in [0, 1] -> CPU uint8 (F, H, W, C); None when the fast path does not apply.+ ``prepare(piece, start)`` maps each raw slice first, keeping float temporaries slice-sized."""import torchdevice = frames.device@@ -40,7 +41,10 @@ def device_uint8(frames: Any) -> Optional[Any]:in_range = Nonetry:for start in range(0, count, step):- piece = frames[start : start + step].to(torch.float32)+ piece = frames[start : start + step]+ if prepare is not None:+ piece = prepare(piece, start)+ piece = piece.to(torch.float32)# Range-checked per slice: a whole-clip reduction over the permuted view copies the clip first.lo, hi = torch.aminmax(piece)fits = (lo >= 0) & (hi <= 1)@@ -175,6 +179,35 @@ def finite_video_decode(pipe: Any, attrs: "tuple[str, ...]" = ("vae",)) -> Iterapass+_FRAME_WISE_METHODS = ("postprocess", "_denormalize_conditionally", "denormalize")+++def _frame_wise_prepare(proc: Any, postprocess: Any, frames: Any, kwargs: dict) -> Any:+ """Per-slice ``postprocess(slice, "pt")`` prepare, or None. Only diffusers' own unoverridden VaeImageProcessor+ methods are frame-wise; a subclass or instance override may mix frames, so it keeps the whole-clip call."""+ try:+ from diffusers.image_processor import VaeImageProcessor+ except Exception: # noqa: BLE001+ return None+ own = vars(proc) if hasattr(proc, "__dict__") else {}+ for name in _FRAME_WISE_METHODS:+ if name in own or getattr(type(proc), name, None) is not getattr(+ VaeImageProcessor, name, None+ ):+ return None+ if set(kwargs) - {"do_denormalize"}:+ return None+ flags = kwargs.get("do_denormalize")+ if flags is None:+ return lambda piece, _start: postprocess(piece, "pt")+ if not isinstance(flags, (list, tuple)) or len(flags) < frames.shape[0]:+ return None+ flags = list(flags)+ return lambda piece, start: postprocess(+ piece, "pt", do_denormalize = flags[start : start + piece.shape[0]]+ )++@contextlib.contextmanagerdef uint8_video_frames(pipe: Any) -> Iterator[None]:"""While active, ``pipe.video_processor``'s np / pil ``postprocess_video`` returns CPU uint8 (B, F, H, W, C), and@@ -207,7 +240,12 @@ def uint8_video_frames(pipe: Any) -> Iterator[None]:clips = []for batch in range(video.shape[0]):# "pt" stops right after the denormalize the np / pil paths apply, on the same view they use.- clip = device_uint8(postprocess(video[batch].permute(1, 0, 2, 3), "pt", **kwargs))+ frames = video[batch].permute(1, 0, 2, 3)+ prepare = _frame_wise_prepare(proc, postprocess, frames, kwargs)+ if prepare is not None:+ clip = device_uint8(frames, prepare)+ else:+ clip = device_uint8(postprocess(frames, "pt", **kwargs))if clip is None:return original(video, output_type, **kwargs)clips.append(clip)diff --git a/studio/backend/tests/test_video_frames.py b/studio/backend/tests/test_video_frames.pyindex e1de5e06c11..5267099245d 100644--- a/studio/backend/tests/test_video_frames.py+++ b/studio/backend/tests/test_video_frames.py@@ -145,3 +145,81 @@ def test_a_card_too_full_for_a_slice_takes_the_original_postprocess(monkeypatch)monkeypatch.setattr(torch, "aminmax", _oom)got = _run(vp, video, "np")assert isinstance(got, np.ndarray) and np.array_equal(got, expected)+++def _denormalize_spy(monkeypatch):+ # Patched on VaeImageProcessor itself, so the processor still resolves to diffusers' own frame-wise method.+ seen = []+ original = VaeImageProcessor._denormalize_conditionally++ def denormalize(+ self,+ images,+ do_denormalize = None,+ ):+ seen.append(int(images.shape[0]))+ return original(self, images, do_denormalize)++ monkeypatch.setattr(VaeImageProcessor, "_denormalize_conditionally", denormalize)+ return seen++[email protected]("dtype", [torch.float32, torch.float16])[email protected]("output_type", ["np", "pil"])+def test_denormalize_runs_per_slice_not_on_the_whole_clip(dtype, output_type, monkeypatch):+ import core.inference.video_frames as vf++ vp = VideoProcessor(vae_scale_factor = 8, do_normalize = True)+ # 9 frames in slices of 2: odd tail, two clips in the batch+ video = torch.cat([_clip(dtype, normalized = True, frames = 9, seed = s) for s in (5, 6)])+ expected = _encode_video_uint8(vp.postprocess_video(video, output_type = "np"))+ monkeypatch.setattr(vf, "_SLICE_BYTES", 2 * 3 * 16 * 24 * 4)+ seen = _denormalize_spy(monkeypatch)+ got = _run(vp, video, output_type)+ assert np.array_equal(got.numpy(), expected)+ assert seen and max(seen) <= 2 and sum(seen) == 18+++def test_per_frame_do_denormalize_list_is_sliced_with_the_frames(monkeypatch):+ import core.inference.video_frames as vf++ vp = VideoProcessor(vae_scale_factor = 8, do_normalize = True)+ video = _clip(torch.float32, normalized = True, frames = 7)+ flags = [True, False, True, True, False, False, True]+ video[0, :, [1, 4, 5]] = video[0, :, [1, 4, 5]].clamp(+ 0, 1+ ) # undenormalized frames stay in range+ expected = _encode_video_uint8(+ vp.postprocess_video(video, output_type = "np", do_denormalize = flags)+ )+ monkeypatch.setattr(vf, "_SLICE_BYTES", 3 * 3 * 16 * 24 * 4)+ seen = _denormalize_spy(monkeypatch)+ with uint8_video_frames(types.SimpleNamespace(video_processor = vp)):+ got = vp.postprocess_video(video, output_type = "np", do_denormalize = flags)+ assert isinstance(got, torch.Tensor) and np.array_equal(got.numpy(), expected)+ assert seen == [3, 3, 1]+++def test_a_processor_that_overrides_postprocess_keeps_the_whole_clip_call(monkeypatch):+ calls = []++ class Mixing(VideoProcessor):+ def postprocess(+ self,+ image,+ output_type = "pil",+ **kwargs,+ ):+ calls.append(int(image.shape[0]))+ return super().postprocess(image, output_type, **kwargs)++ vp = Mixing(vae_scale_factor = 8, do_normalize = True)+ video = _clip(torch.float32, normalized = True, frames = 5)+ expected = _encode_video_uint8(VideoProcessor.postprocess_video(vp, video, output_type = "np"))+ calls.clear()+ import core.inference.video_frames as vf++ monkeypatch.setattr(vf, "_SLICE_BYTES", 3 * 16 * 24 * 4)+ got = _run(vp, video, "np")+ assert np.array_equal(got.numpy(), expected)+ assert calls == [5]