Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio video: export GIFs with one clip-wide palette (about 3x faster) (#14581)

Daniel Han2749eeedaa99
diff --git a/studio/backend/core/inference/video_gallery.py b/studio/backend/core/inference/video_gallery.py
index 26830c6ce10..669f73a3405 100644
--- a/studio/backend/core/inference/video_gallery.py
+++ b/studio/backend/core/inference/video_gallery.py
@@ -16,6 +16,7 @@ import re
import threading
import uuid
from collections.abc import Callable
+from fractions import Fraction
from pathlib import Path
from typing import Any, BinaryIO, Optional, Union
@@ -359,7 +360,6 @@ def transcode_to_file(video_id: str, fmt: str) -> Optional[Path]:
if normalized == "webm":
_transcode_webm(path, dest)
else:
- # GIF is already bounded by _GIF_MAX_FRAMES / _GIF_MAX_EDGE, so it is built in memory and written out.
dest.write_bytes(_transcode_gif(path))
except BaseException:
dest.unlink(missing_ok = True)
@@ -453,66 +453,205 @@ def _transcode_webm(path: Path, dest: Path) -> None:
raise RuntimeError(f"WebM export failed (libvpx-vp9 unavailable?): {exc}") from exc
-# Ceilings for a GIF export, which must hold every kept frame in memory before encoding. 720 px and 300 frames (25s at
-# the 12 fps target) bound that at roughly 150 MB for the widest clip a generate request allows.
_GIF_MAX_EDGE = 720
_GIF_MAX_FRAMES = 300
+_GIF_PALETTEGEN_ARGS = "reserve_transparent=on:transparency_color=ffffff"
+_GIF_PALETTEUSE_ARGS = "dither=sierra2_4a"
+# palettegen needs every frame before paletteuse; above this budget decode twice instead of holding frames.
+_GIF_HOLD_BYTES = 160 * 1024 * 1024
+
+
+def _gif_plan(in_v) -> tuple[Fraction, int, int, int]:
+ """(source fps, frame step, output width, output height) for a GIF of ``in_v``."""
+ rate = Fraction(in_v.average_rate or 24)
+ step = max(1, round(float(rate) / 12))
+ total = in_v.frames or 0
+ kept = (total + step - 1) // step if total else 0
+ if kept > _GIF_MAX_FRAMES:
+ step = -(-total // _GIF_MAX_FRAMES)
+ width, height = in_v.codec_context.width, in_v.codec_context.height
+ if max(width, height) > _GIF_MAX_EDGE:
+ scale = _GIF_MAX_EDGE / max(width, height)
+ width, height = max(1, round(width * scale)), max(1, round(height * scale))
+ return rate, step, width, height
+
+
+def _gif_scaled_frames(path: Path, step: int, width: int, height: int):
+ import av
+ with av.open(str(path)) as src:
+ in_v = src.streams.video[0]
+ in_v.thread_type = "AUTO"
+ kept = 0
+ for i, frame in enumerate(src.decode(in_v)):
+ if i % step:
+ continue
+ if kept >= _GIF_MAX_FRAMES:
+ break
+ kept += 1
+ yield frame.reformat(
+ width = width, height = height, format = "rgb24", interpolation = "LANCZOS"
+ )
-def _transcode_gif(path: Path) -> bytes:
+def _gif_palette_supported() -> bool:
+ try:
+ import av
+ import av.filter
+
+ filters = av.filter.filters_available
+ if "palettegen" not in filters or "paletteuse" not in filters:
+ return False
+ # codecs_available lists a codec usable in either direction; only an encoder writes the GIF
+ av.codec.Codec("gif", "w")
+ return True
+ except Exception: # noqa: BLE001 -- an old or partial build: use the Pillow path
+ return False
+
+
+def _drain_sink(sink) -> list[Any]:
+ import av
+ out = []
+ while True:
+ try:
+ out.append(sink.pull())
+ except (av.error.BlockingIOError, av.error.EOFError):
+ return out
+
+
+def _transcode_gif_palette(
+ path: Path, rate: Fraction, step: int, width: int, height: int, kept: int
+) -> bytes:
+ """GIF via one clip-wide palette (palettegen + paletteuse); per-frame quantizing was slow and flickered."""
import io
+ import av
+ import av.filter
+
+ hold = kept > 0 and kept * width * height * 3 <= _GIF_HOLD_BYTES
+ held: Optional[list[Any]] = [] if hold else None
+ time_base = Fraction(1, 100)
+
+ graph = av.filter.Graph()
+ buffer = graph.add_buffer(width = width, height = height, format = "rgb24", time_base = time_base)
+ palettegen = graph.add("palettegen", _GIF_PALETTEGEN_ARGS)
+ sink = graph.add("buffersink")
+ buffer.link_to(palettegen)
+ palettegen.link_to(sink)
+ graph.configure()
+ count = 0
+ for frame in _gif_scaled_frames(path, step, width, height):
+ frame.pts = count
+ frame.time_base = time_base
+ buffer.push(frame)
+ if held is not None:
+ held.append(frame)
+ count += 1
+ if not count:
+ raise RuntimeError("GIF export decoded no frames.")
+ buffer.push(None)
+ palettes = _drain_sink(sink)
+ if not palettes:
+ raise RuntimeError("GIF export failed: palettegen produced no palette.")
+ palette = palettes[-1]
+
+ graph = av.filter.Graph()
+ frames_in = graph.add_buffer(width = width, height = height, format = "rgb24", time_base = time_base)
+ palette_in = graph.add_buffer(
+ width = palette.width, height = palette.height, format = palette.format.name, time_base = time_base
+ )
+ paletteuse = graph.add("paletteuse", _GIF_PALETTEUSE_ARGS)
+ sink = graph.add("buffersink")
+ frames_in.link_to(paletteuse, 0, 0)
+ palette_in.link_to(paletteuse, 0, 1)
+ paletteuse.link_to(sink)
+ graph.configure()
+ palette.pts = 0
+ palette.time_base = time_base
+ palette_in.push(palette)
+ palette_in.push(None)
+
+ # Frame k at pts k on the rate/step clock; the muxer derives centisecond delays from pts gaps (no drift).
+ out_rate = rate / step
+ buf = io.BytesIO()
+ with av.open(buf, "w", format = "gif", options = {"loop": "0"}) as dst:
+ stream = dst.add_stream("gif", rate = out_rate)
+ stream.width = width
+ stream.height = height
+ stream.pix_fmt = "pal8"
+ written = 0
+
+ def _emit(frames: list[Any]) -> None:
+ nonlocal written
+ for out in frames:
+ out.pts = written
+ out.time_base = 1 / out_rate
+ written += 1
+ for packet in stream.encode(out):
+ packet.duration = 1
+ dst.mux(packet)
+
+ source = held if held is not None else _gif_scaled_frames(path, step, width, height)
+ for k, frame in enumerate(source):
+ if k >= count:
+ break
+ frame.pts = k
+ frame.time_base = time_base
+ frames_in.push(frame)
+ _emit(_drain_sink(sink))
+ frames_in.push(None)
+ _emit(_drain_sink(sink))
+ for packet in stream.encode():
+ packet.duration = 1
+ dst.mux(packet)
+ return buf.getvalue()
+
+
+def _transcode_gif_pillow(path: Path, rate: Fraction, step: int, width: int, height: int) -> bytes:
+ """Fallback for a PyAV build without palettegen / paletteuse / the GIF encoder: Pillow, one palette per frame."""
+ import io
+
+ from PIL import Image
+
+ frames = [
+ frame.to_image().convert("P", palette = Image.Palette.ADAPTIVE)
+ for frame in _gif_scaled_frames(path, step, width, height)
+ ]
+ if not frames:
+ raise RuntimeError("GIF export decoded no frames.")
+ duration_ms = max(20, int(1000 * step / float(rate)))
+ buf = io.BytesIO()
+ frames[0].save(
+ buf,
+ format = "GIF",
+ save_all = True,
+ append_images = frames[1:],
+ duration = duration_ms,
+ loop = 0,
+ )
+ return buf.getvalue()
+
+
+def _transcode_gif(path: Path) -> bytes:
try:
import av
- from PIL import Image
+ from PIL import Image # noqa: F401 -- the fallback encoder; checked up front so the error names it
except Exception as exc: # noqa: BLE001 -- missing deps -> no transcode
raise RuntimeError("GIF export needs the 'av' and 'Pillow' packages.") from exc
- frames: list[Any] = []
try:
with av.open(str(path)) as src:
if not src.streams.video:
raise RuntimeError("GIF export failed: the clip has no video stream.")
in_v = src.streams.video[0]
- rate = float(in_v.average_rate or 24)
- # Full-rate GIFs are huge and stutter; ~12 fps (skipping source frames) is the sweet spot.
- step = max(1, round(rate / 12))
- # Every kept frame is held as a paletted image until the encoder runs, so an unbounded walk is a memory bomb
- # (a 2048x2048 clip of 1024 frames is >4 GB). Bound both axes: downscale past _GIF_MAX_EDGE and widen the
- # step to at most _GIF_MAX_FRAMES. MP4 keeps the full clip.
+ rate, step, width, height = _gif_plan(in_v)
total = in_v.frames or 0
- kept = (total + step - 1) // step if total else 0
- if kept > _GIF_MAX_FRAMES:
- step = -(-total // _GIF_MAX_FRAMES)
- for i, frame in enumerate(src.decode(in_v)):
- if i % step:
- continue
- if len(frames) >= _GIF_MAX_FRAMES:
- break
- image = frame.to_image()
- if max(image.size) > _GIF_MAX_EDGE:
- scale = _GIF_MAX_EDGE / max(image.size)
- image = image.resize(
- (max(1, round(image.width * scale)), max(1, round(image.height * scale))),
- Image.Resampling.LANCZOS,
- )
- frames.append(image.convert("P", palette = Image.Palette.ADAPTIVE))
+ kept = min(_GIF_MAX_FRAMES, (total + step - 1) // step) if total else 0
+ if _gif_palette_supported():
+ return _transcode_gif_palette(path, rate, step, width, height, kept)
+ return _transcode_gif_pillow(path, rate, step, width, height)
except RuntimeError:
raise
except Exception as exc: # noqa: BLE001 -- surface as "decoder unavailable"
raise RuntimeError(f"GIF export failed to decode the clip: {exc}") from exc
- if not frames:
- raise RuntimeError("GIF export decoded no frames.")
- duration_ms = max(20, int(1000 * step / rate))
- buf = io.BytesIO()
- frames[0].save(
- buf,
- format = "GIF",
- save_all = True,
- append_images = frames[1:],
- duration = duration_ms,
- loop = 0,
- )
- return buf.getvalue()
def _sidecar_path(video_id: str) -> Path:
diff --git a/studio/backend/tests/test_video_gallery.py b/studio/backend/tests/test_video_gallery.py
index fc3218667c3..1df3d21e9a1 100644
--- a/studio/backend/tests/test_video_gallery.py
+++ b/studio/backend/tests/test_video_gallery.py
@@ -625,6 +625,190 @@ def test_gif_export_bounds_frames_and_edge(monkeypatch):
assert frames <= 4, frames
+def _gif_blocks(data: bytes) -> dict:
+ """Walk a GIF's blocks: loop count, per-frame delays (cs) and whether each frame carries a local palette."""
+ assert data[:6] in (b"GIF87a", b"GIF89a")
+ width, height, packed = (
+ int.from_bytes(data[6:8], "little"),
+ int.from_bytes(data[8:10], "little"),
+ data[10],
+ )
+ pos = 13 + (3 << ((packed & 7) + 1) if packed & 0x80 else 0)
+ info = {
+ "size": (width, height),
+ "global_palette": bool(packed & 0x80),
+ "loop": None,
+ "delays": [],
+ "local": [],
+ }
+
+ def _skip_sub_blocks(i: int) -> int:
+ while data[i]:
+ i += data[i] + 1
+ return i + 1
+
+ while data[pos] != 0x3B:
+ if data[pos] == 0x21:
+ label = data[pos + 1]
+ if label == 0xF9:
+ info["delays"].append(int.from_bytes(data[pos + 4 : pos + 6], "little"))
+ elif label == 0xFF and data[pos + 3 : pos + 14] == b"NETSCAPE2.0":
+ info["loop"] = int.from_bytes(data[pos + 16 : pos + 18], "little")
+ pos = _skip_sub_blocks(pos + 2)
+ elif data[pos] == 0x2C:
+ flags = data[pos + 9]
+ info["local"].append(bool(flags & 0x80))
+ pos += 10 + (3 << ((flags & 7) + 1) if flags & 0x80 else 0)
+ pos = _skip_sub_blocks(pos + 1)
+ else:
+ raise AssertionError(f"bad GIF block 0x{data[pos]:02x} at {pos}")
+ return info
+
+
+def _gradient_mp4_bytes(frames: int, size: tuple[int, int], rate: int) -> bytes:
+ av = pytest.importorskip("av")
+ np = pytest.importorskip("numpy")
+ import io
+
+ width, height = size
+ yy, xx = np.mgrid[0:height, 0:width]
+ buf = io.BytesIO()
+ with av.open(buf, "w", format = "mp4") as out:
+ stream = out.add_stream("mpeg4", rate = rate)
+ stream.width, stream.height, stream.pix_fmt = width, height, "yuv420p"
+ for i in range(frames):
+ rgb = np.stack(
+ [
+ (xx * 255 // width + i * 7) % 256,
+ (yy * 255 // height) % 256,
+ (xx + yy + i * 11) % 256,
+ ],
+ axis = -1,
+ ).astype(np.uint8)
+ for packet in stream.encode(av.VideoFrame.from_ndarray(rgb, format = "rgb24")):
+ out.mux(packet)
+ for packet in stream.encode():
+ out.mux(packet)
+ return buf.getvalue()
+
+
+def _count_calls(monkeypatch, module, name: str) -> dict:
+ calls = {"n": 0}
+ real = getattr(module, name)
+
+ def _spy(*args, **kwargs):
+ calls["n"] += 1
+ return real(*args, **kwargs)
+
+ monkeypatch.setattr(module, name, _spy)
+ return calls
+
+
[email protected]("hold", [True, False], ids = ["held_frames", "two_decode_passes"])
+def test_gif_export_uses_one_clip_wide_palette(monkeypatch, hold):
+ from core.inference import video_gallery as vg
+
+ pytest.importorskip("PIL.Image")
+ if not vg._gif_palette_supported():
+ pytest.skip("this PyAV build has no palettegen / paletteuse / gif encoder")
+ if not hold:
+ monkeypatch.setattr(vg, "_GIF_HOLD_BYTES", 0)
+ palette_calls = _count_calls(monkeypatch, vg, "_transcode_gif_palette")
+ pillow_calls = _count_calls(monkeypatch, vg, "_transcode_gif_pillow")
+ decodes = _count_calls(monkeypatch, vg, "_gif_scaled_frames")
+
+ record = gallery.save(_gradient_mp4_bytes(frames = 24, size = (96, 64), rate = 24), _meta())
+ gif = vg._transcode_gif(vg.gallery_dir() / f"{record['id']}.mp4")
+
+ assert palette_calls["n"] == 1 and pillow_calls["n"] == 0
+ assert decodes["n"] == (1 if hold else 2)
+ info = _gif_blocks(gif)
+ assert info["size"] == (96, 64)
+ assert info["global_palette"] and not any(info["local"][1:]), info["local"]
+ assert len(info["local"]) == 12
+ assert info["loop"] == 0 # NETSCAPE loop 0 = forever
+ assert all(d in (8, 9) for d in info["delays"]), info["delays"]
+ assert sum(info["delays"]) in (99, 100, 101)
+
+
+def test_gif_export_palette_path_caps_frames_and_edge(monkeypatch):
+ from core.inference import video_gallery as vg
+
+ if not vg._gif_palette_supported():
+ pytest.skip("this PyAV build has no palettegen / paletteuse / gif encoder")
+ monkeypatch.setattr(vg, "_GIF_MAX_EDGE", 48)
+ monkeypatch.setattr(vg, "_GIF_MAX_FRAMES", 5)
+ record = gallery.save(_gradient_mp4_bytes(frames = 30, size = (96, 64), rate = 12), _meta())
+ info = _gif_blocks(vg._transcode_gif(vg.gallery_dir() / f"{record['id']}.mp4"))
+ assert info["size"] == (48, 32)
+ assert 1 <= len(info["local"]) <= 5
+ assert info["loop"] == 0
+
+
+def test_gif_export_falls_back_to_pillow_without_the_palette_filters(monkeypatch):
+ import av.filter
+
+ from core.inference import video_gallery as vg
+
+ pytest.importorskip("PIL.Image")
+ monkeypatch.setattr(
+ av.filter,
+ "filters_available",
+ set(av.filter.filters_available) - {"palettegen", "paletteuse"},
+ )
+ assert vg._gif_palette_supported() is False
+ palette_calls = _count_calls(monkeypatch, vg, "_transcode_gif_palette")
+ pillow_calls = _count_calls(monkeypatch, vg, "_transcode_gif_pillow")
+
+ record = gallery.save(_gradient_mp4_bytes(frames = 24, size = (96, 64), rate = 24), _meta())
+ info = _gif_blocks(vg._transcode_gif(vg.gallery_dir() / f"{record['id']}.mp4"))
+
+ assert palette_calls["n"] == 0 and pillow_calls["n"] == 1
+ assert info["size"] == (96, 64) and len(info["local"]) == 12 and info["loop"] == 0
+ assert all(d == 8 for d in info["delays"]), info["delays"]
+
+
+def test_gif_export_falls_back_to_pillow_without_a_gif_encoder(monkeypatch):
+ import av.codec
+
+ from core.inference import video_gallery as vg
+
+ pytest.importorskip("PIL.Image")
+ real_codec = av.codec.Codec
+
+ def _decoder_only(name, mode = "r"):
+ if name == "gif" and mode == "w":
+ raise av.codec.codec.UnknownCodecError(name)
+ return real_codec(name, mode)
+
+ monkeypatch.setattr(av.codec, "Codec", _decoder_only)
+ assert "gif" in av.codecs_available
+ assert vg._gif_palette_supported() is False
+ pillow_calls = _count_calls(monkeypatch, vg, "_transcode_gif_pillow")
+
+ record = gallery.save(_gradient_mp4_bytes(frames = 24, size = (96, 64), rate = 24), _meta())
+ info = _gif_blocks(vg._transcode_gif(vg.gallery_dir() / f"{record['id']}.mp4"))
+
+ assert pillow_calls["n"] == 1 and info["loop"] == 0
+
+
+def test_gif_export_missing_deps_keeps_the_error_message(monkeypatch):
+ import builtins
+
+ from core.inference import video_gallery as vg
+
+ real_import = builtins.__import__
+
+ def _no_av(name, *args, **kwargs):
+ if name == "av" or name.startswith("av."):
+ raise ImportError("no av")
+ return real_import(name, *args, **kwargs)
+
+ monkeypatch.setattr(builtins, "__import__", _no_av)
+ with pytest.raises(RuntimeError, match = "GIF export needs the 'av' and 'Pillow' packages."):
+ vg._transcode_gif(vg.gallery_dir() / "missing.mp4")
+
+
# --- pin / archive flags ---------------------------------------------------------------------