Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio video: export GIFs with one clip-wide palette (about 3x faster) (#14581)
Daniel Han2749eeedaa99
diff --git a/studio/backend/core/inference/video_gallery.py b/studio/backend/core/inference/video_gallery.pyindex 26830c6ce10..669f73a3405 100644--- a/studio/backend/core/inference/video_gallery.py+++ b/studio/backend/core/inference/video_gallery.py@@ -16,6 +16,7 @@ import reimport threadingimport uuidfrom collections.abc import Callable+from fractions import Fractionfrom pathlib import Pathfrom typing import Any, BinaryIO, Optional, Union@@ -359,7 +360,6 @@ def transcode_to_file(video_id: str, fmt: str) -> Optional[Path]:if normalized == "webm":_transcode_webm(path, dest)else:- # GIF is already bounded by _GIF_MAX_FRAMES / _GIF_MAX_EDGE, so it is built in memory and written out.dest.write_bytes(_transcode_gif(path))except BaseException:dest.unlink(missing_ok = True)@@ -453,66 +453,205 @@ def _transcode_webm(path: Path, dest: Path) -> None:raise RuntimeError(f"WebM export failed (libvpx-vp9 unavailable?): {exc}") from exc-# Ceilings for a GIF export, which must hold every kept frame in memory before encoding. 720 px and 300 frames (25s at-# the 12 fps target) bound that at roughly 150 MB for the widest clip a generate request allows._GIF_MAX_EDGE = 720_GIF_MAX_FRAMES = 300+_GIF_PALETTEGEN_ARGS = "reserve_transparent=on:transparency_color=ffffff"+_GIF_PALETTEUSE_ARGS = "dither=sierra2_4a"+# palettegen needs every frame before paletteuse; above this budget decode twice instead of holding frames.+_GIF_HOLD_BYTES = 160 * 1024 * 1024+++def _gif_plan(in_v) -> tuple[Fraction, int, int, int]:+ """(source fps, frame step, output width, output height) for a GIF of ``in_v``."""+ rate = Fraction(in_v.average_rate or 24)+ step = max(1, round(float(rate) / 12))+ total = in_v.frames or 0+ kept = (total + step - 1) // step if total else 0+ if kept > _GIF_MAX_FRAMES:+ step = -(-total // _GIF_MAX_FRAMES)+ width, height = in_v.codec_context.width, in_v.codec_context.height+ if max(width, height) > _GIF_MAX_EDGE:+ scale = _GIF_MAX_EDGE / max(width, height)+ width, height = max(1, round(width * scale)), max(1, round(height * scale))+ return rate, step, width, height+++def _gif_scaled_frames(path: Path, step: int, width: int, height: int):+ import av+ with av.open(str(path)) as src:+ in_v = src.streams.video[0]+ in_v.thread_type = "AUTO"+ kept = 0+ for i, frame in enumerate(src.decode(in_v)):+ if i % step:+ continue+ if kept >= _GIF_MAX_FRAMES:+ break+ kept += 1+ yield frame.reformat(+ width = width, height = height, format = "rgb24", interpolation = "LANCZOS"+ )-def _transcode_gif(path: Path) -> bytes:+def _gif_palette_supported() -> bool:+ try:+ import av+ import av.filter++ filters = av.filter.filters_available+ if "palettegen" not in filters or "paletteuse" not in filters:+ return False+ # codecs_available lists a codec usable in either direction; only an encoder writes the GIF+ av.codec.Codec("gif", "w")+ return True+ except Exception: # noqa: BLE001 -- an old or partial build: use the Pillow path+ return False+++def _drain_sink(sink) -> list[Any]:+ import av+ out = []+ while True:+ try:+ out.append(sink.pull())+ except (av.error.BlockingIOError, av.error.EOFError):+ return out+++def _transcode_gif_palette(+ path: Path, rate: Fraction, step: int, width: int, height: int, kept: int+) -> bytes:+ """GIF via one clip-wide palette (palettegen + paletteuse); per-frame quantizing was slow and flickered."""import io+ import av+ import av.filter++ hold = kept > 0 and kept * width * height * 3 <= _GIF_HOLD_BYTES+ held: Optional[list[Any]] = [] if hold else None+ time_base = Fraction(1, 100)++ graph = av.filter.Graph()+ buffer = graph.add_buffer(width = width, height = height, format = "rgb24", time_base = time_base)+ palettegen = graph.add("palettegen", _GIF_PALETTEGEN_ARGS)+ sink = graph.add("buffersink")+ buffer.link_to(palettegen)+ palettegen.link_to(sink)+ graph.configure()+ count = 0+ for frame in _gif_scaled_frames(path, step, width, height):+ frame.pts = count+ frame.time_base = time_base+ buffer.push(frame)+ if held is not None:+ held.append(frame)+ count += 1+ if not count:+ raise RuntimeError("GIF export decoded no frames.")+ buffer.push(None)+ palettes = _drain_sink(sink)+ if not palettes:+ raise RuntimeError("GIF export failed: palettegen produced no palette.")+ palette = palettes[-1]++ graph = av.filter.Graph()+ frames_in = graph.add_buffer(width = width, height = height, format = "rgb24", time_base = time_base)+ palette_in = graph.add_buffer(+ width = palette.width, height = palette.height, format = palette.format.name, time_base = time_base+ )+ paletteuse = graph.add("paletteuse", _GIF_PALETTEUSE_ARGS)+ sink = graph.add("buffersink")+ frames_in.link_to(paletteuse, 0, 0)+ palette_in.link_to(paletteuse, 0, 1)+ paletteuse.link_to(sink)+ graph.configure()+ palette.pts = 0+ palette.time_base = time_base+ palette_in.push(palette)+ palette_in.push(None)++ # Frame k at pts k on the rate/step clock; the muxer derives centisecond delays from pts gaps (no drift).+ out_rate = rate / step+ buf = io.BytesIO()+ with av.open(buf, "w", format = "gif", options = {"loop": "0"}) as dst:+ stream = dst.add_stream("gif", rate = out_rate)+ stream.width = width+ stream.height = height+ stream.pix_fmt = "pal8"+ written = 0++ def _emit(frames: list[Any]) -> None:+ nonlocal written+ for out in frames:+ out.pts = written+ out.time_base = 1 / out_rate+ written += 1+ for packet in stream.encode(out):+ packet.duration = 1+ dst.mux(packet)++ source = held if held is not None else _gif_scaled_frames(path, step, width, height)+ for k, frame in enumerate(source):+ if k >= count:+ break+ frame.pts = k+ frame.time_base = time_base+ frames_in.push(frame)+ _emit(_drain_sink(sink))+ frames_in.push(None)+ _emit(_drain_sink(sink))+ for packet in stream.encode():+ packet.duration = 1+ dst.mux(packet)+ return buf.getvalue()+++def _transcode_gif_pillow(path: Path, rate: Fraction, step: int, width: int, height: int) -> bytes:+ """Fallback for a PyAV build without palettegen / paletteuse / the GIF encoder: Pillow, one palette per frame."""+ import io++ from PIL import Image++ frames = [+ frame.to_image().convert("P", palette = Image.Palette.ADAPTIVE)+ for frame in _gif_scaled_frames(path, step, width, height)+ ]+ if not frames:+ raise RuntimeError("GIF export decoded no frames.")+ duration_ms = max(20, int(1000 * step / float(rate)))+ buf = io.BytesIO()+ frames[0].save(+ buf,+ format = "GIF",+ save_all = True,+ append_images = frames[1:],+ duration = duration_ms,+ loop = 0,+ )+ return buf.getvalue()+++def _transcode_gif(path: Path) -> bytes:try:import av- from PIL import Image+ from PIL import Image # noqa: F401 -- the fallback encoder; checked up front so the error names itexcept Exception as exc: # noqa: BLE001 -- missing deps -> no transcoderaise RuntimeError("GIF export needs the 'av' and 'Pillow' packages.") from exc- frames: list[Any] = []try:with av.open(str(path)) as src:if not src.streams.video:raise RuntimeError("GIF export failed: the clip has no video stream.")in_v = src.streams.video[0]- rate = float(in_v.average_rate or 24)- # Full-rate GIFs are huge and stutter; ~12 fps (skipping source frames) is the sweet spot.- step = max(1, round(rate / 12))- # Every kept frame is held as a paletted image until the encoder runs, so an unbounded walk is a memory bomb- # (a 2048x2048 clip of 1024 frames is >4 GB). Bound both axes: downscale past _GIF_MAX_EDGE and widen the- # step to at most _GIF_MAX_FRAMES. MP4 keeps the full clip.+ rate, step, width, height = _gif_plan(in_v)total = in_v.frames or 0- kept = (total + step - 1) // step if total else 0- if kept > _GIF_MAX_FRAMES:- step = -(-total // _GIF_MAX_FRAMES)- for i, frame in enumerate(src.decode(in_v)):- if i % step:- continue- if len(frames) >= _GIF_MAX_FRAMES:- break- image = frame.to_image()- if max(image.size) > _GIF_MAX_EDGE:- scale = _GIF_MAX_EDGE / max(image.size)- image = image.resize(- (max(1, round(image.width * scale)), max(1, round(image.height * scale))),- Image.Resampling.LANCZOS,- )- frames.append(image.convert("P", palette = Image.Palette.ADAPTIVE))+ kept = min(_GIF_MAX_FRAMES, (total + step - 1) // step) if total else 0+ if _gif_palette_supported():+ return _transcode_gif_palette(path, rate, step, width, height, kept)+ return _transcode_gif_pillow(path, rate, step, width, height)except RuntimeError:raiseexcept Exception as exc: # noqa: BLE001 -- surface as "decoder unavailable"raise RuntimeError(f"GIF export failed to decode the clip: {exc}") from exc- if not frames:- raise RuntimeError("GIF export decoded no frames.")- duration_ms = max(20, int(1000 * step / rate))- buf = io.BytesIO()- frames[0].save(- buf,- format = "GIF",- save_all = True,- append_images = frames[1:],- duration = duration_ms,- loop = 0,- )- return buf.getvalue()def _sidecar_path(video_id: str) -> Path:diff --git a/studio/backend/tests/test_video_gallery.py b/studio/backend/tests/test_video_gallery.pyindex fc3218667c3..1df3d21e9a1 100644--- a/studio/backend/tests/test_video_gallery.py+++ b/studio/backend/tests/test_video_gallery.py@@ -625,6 +625,190 @@ def test_gif_export_bounds_frames_and_edge(monkeypatch):assert frames <= 4, frames+def _gif_blocks(data: bytes) -> dict:+ """Walk a GIF's blocks: loop count, per-frame delays (cs) and whether each frame carries a local palette."""+ assert data[:6] in (b"GIF87a", b"GIF89a")+ width, height, packed = (+ int.from_bytes(data[6:8], "little"),+ int.from_bytes(data[8:10], "little"),+ data[10],+ )+ pos = 13 + (3 << ((packed & 7) + 1) if packed & 0x80 else 0)+ info = {+ "size": (width, height),+ "global_palette": bool(packed & 0x80),+ "loop": None,+ "delays": [],+ "local": [],+ }++ def _skip_sub_blocks(i: int) -> int:+ while data[i]:+ i += data[i] + 1+ return i + 1++ while data[pos] != 0x3B:+ if data[pos] == 0x21:+ label = data[pos + 1]+ if label == 0xF9:+ info["delays"].append(int.from_bytes(data[pos + 4 : pos + 6], "little"))+ elif label == 0xFF and data[pos + 3 : pos + 14] == b"NETSCAPE2.0":+ info["loop"] = int.from_bytes(data[pos + 16 : pos + 18], "little")+ pos = _skip_sub_blocks(pos + 2)+ elif data[pos] == 0x2C:+ flags = data[pos + 9]+ info["local"].append(bool(flags & 0x80))+ pos += 10 + (3 << ((flags & 7) + 1) if flags & 0x80 else 0)+ pos = _skip_sub_blocks(pos + 1)+ else:+ raise AssertionError(f"bad GIF block 0x{data[pos]:02x} at {pos}")+ return info+++def _gradient_mp4_bytes(frames: int, size: tuple[int, int], rate: int) -> bytes:+ av = pytest.importorskip("av")+ np = pytest.importorskip("numpy")+ import io++ width, height = size+ yy, xx = np.mgrid[0:height, 0:width]+ buf = io.BytesIO()+ with av.open(buf, "w", format = "mp4") as out:+ stream = out.add_stream("mpeg4", rate = rate)+ stream.width, stream.height, stream.pix_fmt = width, height, "yuv420p"+ for i in range(frames):+ rgb = np.stack(+ [+ (xx * 255 // width + i * 7) % 256,+ (yy * 255 // height) % 256,+ (xx + yy + i * 11) % 256,+ ],+ axis = -1,+ ).astype(np.uint8)+ for packet in stream.encode(av.VideoFrame.from_ndarray(rgb, format = "rgb24")):+ out.mux(packet)+ for packet in stream.encode():+ out.mux(packet)+ return buf.getvalue()+++def _count_calls(monkeypatch, module, name: str) -> dict:+ calls = {"n": 0}+ real = getattr(module, name)++ def _spy(*args, **kwargs):+ calls["n"] += 1+ return real(*args, **kwargs)++ monkeypatch.setattr(module, name, _spy)+ return calls++[email protected]("hold", [True, False], ids = ["held_frames", "two_decode_passes"])+def test_gif_export_uses_one_clip_wide_palette(monkeypatch, hold):+ from core.inference import video_gallery as vg++ pytest.importorskip("PIL.Image")+ if not vg._gif_palette_supported():+ pytest.skip("this PyAV build has no palettegen / paletteuse / gif encoder")+ if not hold:+ monkeypatch.setattr(vg, "_GIF_HOLD_BYTES", 0)+ palette_calls = _count_calls(monkeypatch, vg, "_transcode_gif_palette")+ pillow_calls = _count_calls(monkeypatch, vg, "_transcode_gif_pillow")+ decodes = _count_calls(monkeypatch, vg, "_gif_scaled_frames")++ record = gallery.save(_gradient_mp4_bytes(frames = 24, size = (96, 64), rate = 24), _meta())+ gif = vg._transcode_gif(vg.gallery_dir() / f"{record['id']}.mp4")++ assert palette_calls["n"] == 1 and pillow_calls["n"] == 0+ assert decodes["n"] == (1 if hold else 2)+ info = _gif_blocks(gif)+ assert info["size"] == (96, 64)+ assert info["global_palette"] and not any(info["local"][1:]), info["local"]+ assert len(info["local"]) == 12+ assert info["loop"] == 0 # NETSCAPE loop 0 = forever+ assert all(d in (8, 9) for d in info["delays"]), info["delays"]+ assert sum(info["delays"]) in (99, 100, 101)+++def test_gif_export_palette_path_caps_frames_and_edge(monkeypatch):+ from core.inference import video_gallery as vg++ if not vg._gif_palette_supported():+ pytest.skip("this PyAV build has no palettegen / paletteuse / gif encoder")+ monkeypatch.setattr(vg, "_GIF_MAX_EDGE", 48)+ monkeypatch.setattr(vg, "_GIF_MAX_FRAMES", 5)+ record = gallery.save(_gradient_mp4_bytes(frames = 30, size = (96, 64), rate = 12), _meta())+ info = _gif_blocks(vg._transcode_gif(vg.gallery_dir() / f"{record['id']}.mp4"))+ assert info["size"] == (48, 32)+ assert 1 <= len(info["local"]) <= 5+ assert info["loop"] == 0+++def test_gif_export_falls_back_to_pillow_without_the_palette_filters(monkeypatch):+ import av.filter++ from core.inference import video_gallery as vg++ pytest.importorskip("PIL.Image")+ monkeypatch.setattr(+ av.filter,+ "filters_available",+ set(av.filter.filters_available) - {"palettegen", "paletteuse"},+ )+ assert vg._gif_palette_supported() is False+ palette_calls = _count_calls(monkeypatch, vg, "_transcode_gif_palette")+ pillow_calls = _count_calls(monkeypatch, vg, "_transcode_gif_pillow")++ record = gallery.save(_gradient_mp4_bytes(frames = 24, size = (96, 64), rate = 24), _meta())+ info = _gif_blocks(vg._transcode_gif(vg.gallery_dir() / f"{record['id']}.mp4"))++ assert palette_calls["n"] == 0 and pillow_calls["n"] == 1+ assert info["size"] == (96, 64) and len(info["local"]) == 12 and info["loop"] == 0+ assert all(d == 8 for d in info["delays"]), info["delays"]+++def test_gif_export_falls_back_to_pillow_without_a_gif_encoder(monkeypatch):+ import av.codec++ from core.inference import video_gallery as vg++ pytest.importorskip("PIL.Image")+ real_codec = av.codec.Codec++ def _decoder_only(name, mode = "r"):+ if name == "gif" and mode == "w":+ raise av.codec.codec.UnknownCodecError(name)+ return real_codec(name, mode)++ monkeypatch.setattr(av.codec, "Codec", _decoder_only)+ assert "gif" in av.codecs_available+ assert vg._gif_palette_supported() is False+ pillow_calls = _count_calls(monkeypatch, vg, "_transcode_gif_pillow")++ record = gallery.save(_gradient_mp4_bytes(frames = 24, size = (96, 64), rate = 24), _meta())+ info = _gif_blocks(vg._transcode_gif(vg.gallery_dir() / f"{record['id']}.mp4"))++ assert pillow_calls["n"] == 1 and info["loop"] == 0+++def test_gif_export_missing_deps_keeps_the_error_message(monkeypatch):+ import builtins++ from core.inference import video_gallery as vg++ real_import = builtins.__import__++ def _no_av(name, *args, **kwargs):+ if name == "av" or name.startswith("av."):+ raise ImportError("no av")+ return real_import(name, *args, **kwargs)++ monkeypatch.setattr(builtins, "__import__", _no_av)+ with pytest.raises(RuntimeError, match = "GIF export needs the 'av' and 'Pillow' packages."):+ vg._transcode_gif(vg.gallery_dir() / "missing.mp4")++# --- pin / archive flags ---------------------------------------------------------------------