Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio video: decode reference audio once per request and stage H3 reference frames faster (#16324)

Daniel Hane023bac16dd8
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 2af396d18dc..1e33e127324 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -11795,6 +11795,7 @@ class VideoBackend:
flow_shift: Optional[float] = None,
audio_flow_shift: Optional[float] = None,
video_id: Optional[str] = None,
+ decoded_reference_audios: Optional[Any] = None,
expected_state: Optional[object] = None,
live_preview: Optional[bool] = None,
before_commit: Optional[Callable[[], None]] = None,
@@ -11836,6 +11837,7 @@ class VideoBackend:
reference_image_size,
canvas_w,
canvas_h,
+ decoded_audios = decoded_reference_audios,
)
shift, audio_shift = self._resolve_flow_shifts(
state.family, state.engine, flow_shift, audio_flow_shift
@@ -13159,6 +13161,8 @@ class VideoBackend:
reference_image_size: Optional[str],
width: int,
height: int,
+ *,
+ decoded_audios: Optional[Any] = None,
) -> Any:
"""Sizing is settled here so both engines' min(1, ...) scales are no-ops; native refuses max."""
from .video_minimax_h3 import (
@@ -13231,13 +13235,16 @@ class VideoBackend:
"silent earlier video. Reorder the videos so ones with soundtracks "
"come first, or add a replacement soundtrack to each earlier video."
)
- decoded_audios = tuple(
- decode_h3_reference_audio(_decode_b64_media(item)) for item in audios
- )
+ # The route's readability check may have decoded these exact inputs already.
+ reused_audios = decoded_audios.for_inputs(audios) if decoded_audios is not None else None
+ if reused_audios is None:
+ reused_audios = tuple(
+ decode_h3_reference_audio(_decode_b64_media(item)) for item in audios
+ )
return MiniMaxH3References(
images = fitted_images,
videos = tuple(decoded_videos),
- audios = decoded_audios,
+ audios = reused_audios,
)
def _h3_native_server_render(
diff --git a/studio/backend/core/inference/video_minimax_h3.py b/studio/backend/core/inference/video_minimax_h3.py
index 7704b02c3ca..87a81bf94c4 100644
--- a/studio/backend/core/inference/video_minimax_h3.py
+++ b/studio/backend/core/inference/video_minimax_h3.py
@@ -605,7 +605,9 @@ def _fit_h3_video_frame(
) -> tuple[Any, tuple[int, int]]:
from PIL import Image
- image = frame.to_image().convert("RGB")
+ image = frame.to_image()
+ if image.mode != "RGB": # PyAV already returns a fresh RGB image; convert only copies it
+ image = image.convert("RGB")
if fitted_size is None:
fitted_size = h3_reference_frame_size(*image.size)
if image.size != fitted_size:
@@ -974,6 +976,17 @@ class MiniMaxH3References:
return len(self.images) + len(self.videos) + len(self.audios)
+@dataclass(frozen = True)
+class DecodedReferenceAudios:
+ """Untrimmed ``decode_h3_reference_audio`` results for ``sources``, reused only for those exact inputs."""
+
+ sources: tuple = ()
+ decoded: tuple = ()
+
+ def for_inputs(self, audios: list) -> Optional[tuple]:
+ return self.decoded if tuple(audios) == self.sources else None
+
+
@dataclass(frozen = True)
class MiniMaxH3StagedReferences:
"""``MiniMaxH3References`` written to disk the way sd-cli reads them back."""
@@ -985,6 +998,46 @@ class MiniMaxH3StagedReferences:
audios: tuple[str, ...] = ()
+# Pillow's zlib encoder releases the GIL, so a few threads write a reference video's PNGs in parallel.
+H3_REF_FRAME_WRITERS = 4
+
+
+def _write_h3_reference_frames(directory: Path, frames: Any) -> None:
+ """Write ``frames`` as 00000.png onward, encoding a repeated frame object (24 fps fill) once."""
+ import io
+ from concurrent.futures import ThreadPoolExecutor
+
+ # Keyed by identity: ``frames`` holds every object, so no id is reused while this runs.
+ slots: dict[int, tuple[Any, list[Path]]] = {}
+ for frame_index, frame in enumerate(frames):
+ slots.setdefault(id(frame), (frame, []))[1].append(directory / f"{frame_index:05d}.png")
+
+ def write(slot: tuple[Any, list[Path]]) -> None:
+ frame, paths = slot
+ if len(paths) == 1:
+ frame.save(paths[0], format = "PNG", compress_level = 1)
+ return
+ buffer = io.BytesIO()
+ frame.save(buffer, format = "PNG", compress_level = 1)
+ data = buffer.getvalue()
+ for path in paths:
+ path.write_bytes(data)
+
+ work = list(slots.values())
+ workers = min(H3_REF_FRAME_WRITERS, len(work), os.cpu_count() or 1)
+ if workers <= 1:
+ for slot in work:
+ write(slot)
+ return
+ pool = ThreadPoolExecutor(max_workers = workers, thread_name_prefix = "h3-ref-frames")
+ try:
+ futures = [pool.submit(write, slot) for slot in work]
+ for future in futures:
+ future.result() # first failure in frame order, as the sequential loop raised
+ finally:
+ pool.shutdown(wait = True, cancel_futures = True)
+
+
def stage_h3_references(
references: MiniMaxH3References, scratch: Path
) -> MiniMaxH3StagedReferences:
@@ -1002,8 +1055,7 @@ def stage_h3_references(
for index, (frames, waveform, sample_rate) in enumerate(references.videos):
directory = scratch / f"ref-video-{index:02d}"
directory.mkdir()
- for frame_index, frame in enumerate(frames):
- frame.save(directory / f"{frame_index:05d}.png", format = "PNG", compress_level = 1)
+ _write_h3_reference_frames(directory, frames)
videos.append(str(directory))
if waveform is None:
# Positional pairing cannot express "skip this one", so stop pairing here.
diff --git a/studio/backend/routes/video.py b/studio/backend/routes/video.py
index b025223b26b..a9ba657e2ed 100644
--- a/studio/backend/routes/video.py
+++ b/studio/backend/routes/video.py
@@ -1942,25 +1942,32 @@ def _audio_ref_data_url(ref) -> str:
)
-def _refuse_unreadable_audio(audios: Optional[list[str]]) -> None:
- """Decode each reference audio the engine would, refusing one it could not read."""
+def _refuse_unreadable_audio(audios: Optional[list[str]]):
+ """Decode each reference audio the engine would, refusing one it could not read.
+
+ Returns the decoded audios for this request's render to reuse, or None when nothing was decoded."""
if not audios:
- return
+ return None
from core.inference.video import _decode_b64_media
try:
- from core.inference.video_minimax_h3 import decode_h3_reference_audio
+ from core.inference.video_minimax_h3 import (
+ DecodedReferenceAudios,
+ decode_h3_reference_audio,
+ )
except ImportError: # no media decoder here; begin_generate reports it
- return
+ return None
+ decoded = []
for index, item in enumerate(audios):
try:
- decode_h3_reference_audio(_decode_b64_media(item))
+ decoded.append(decode_h3_reference_audio(_decode_b64_media(item)))
except ImportError:
- return
+ return None
except Exception: # noqa: BLE001 -- any decode failure is client input feedback
raise _openai_video_error(
400, f"reference_audios[{index}] is not readable audio.", param = "reference_audios"
)
+ return DecodedReferenceAudios(sources = tuple(audios), decoded = tuple(decoded))
def _conditioning_param(
@@ -2264,6 +2271,7 @@ async def _start_openai_video(
size, seconds, reference_audios = inputs.size, inputs.seconds, inputs.reference_audios
width, height = size if size is not None else (None, None)
audio_checked = [False]
+ decoded_audios = [None]
def _refuse_unservable_request(pick) -> None:
from core.inference.media_model_index import expected_partition
@@ -2274,7 +2282,7 @@ async def _start_openai_video(
return
_refuse_family_inputs(fam, expected_partition(pick), body, reference, inputs)
# Decoded here, like the images above, so unreadable audio never costs a switch.
- _refuse_unreadable_audio(reference_audios)
+ decoded_audios[0] = _refuse_unreadable_audio(reference_audios)
audio_checked[0] = True
pin_requested_model = bool(
@@ -2297,7 +2305,7 @@ async def _start_openai_video(
)
if not audio_checked[0]:
# No switch ran the check, and the engine's own decode can fail with more than ValueError.
- await asyncio.to_thread(_refuse_unreadable_audio, reference_audios)
+ decoded_audios[0] = await asyncio.to_thread(_refuse_unreadable_audio, reference_audios)
backend = get_video_backend()
# Managed callers are authorized against the exact resident state, pinned to the reservation;
@@ -2353,6 +2361,8 @@ async def _start_openai_video(
reference_audios = reference_audios,
)
generate_kwargs.update({k: v for k, v in controls.items() if v is not None})
+ if decoded_audios[0] is not None:
+ generate_kwargs["decoded_reference_audios"] = decoded_audios[0]
if expected_state is not None:
generate_kwargs["expected_state"] = expected_state
resolved = await asyncio.to_thread(
diff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.py
index 5c5bda27d4d..45ba36086f8 100644
--- a/studio/backend/tests/test_video_backend.py
+++ b/studio/backend/tests/test_video_backend.py
@@ -13938,3 +13938,138 @@ def test_h3_unified_guard_reruns_before_a_failed_hosted_encoder_goes_dense(
assert free_mib < initial_memory.free_mib
assert snapshots == [initial_memory]
assert calls == [("fp8", "int8", initial_memory), ("fp8", None, initial_memory)]
+
+
+def _png_data_url(width = 64, height = 64):
+ import base64
+ import io
+
+ from PIL import Image
+
+ buffer = io.BytesIO()
+ Image.new("RGB", (width, height), (10, 20, 30)).save(buffer, "PNG")
+ return "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode("ascii")
+
+
+def test_h3_render_reuses_the_audio_the_route_already_decoded(monkeypatch):
+ """The readability check's decode is handed to the render only for the same inputs."""
+ pytest.importorskip("av")
+ import numpy as np
+
+ from core.inference import video_minimax_h3 as h3
+ from core.inference.video import VideoBackend
+ from core.inference.video_families import detect_video_family
+ from routes.video import _refuse_unreadable_audio
+
+ first = _data_url_wav(seconds = 1.0, rate = 32_000)
+ second = _data_url_wav(seconds = 2.0, rate = 32_000)
+ real_decode = h3.decode_h3_reference_audio
+ calls = []
+
+ def counting_decode(*args, **kwargs):
+ calls.append(1)
+ return real_decode(*args, **kwargs)
+
+ monkeypatch.setattr(h3, "decode_h3_reference_audio", counting_decode)
+
+ def resolve(audios, decoded):
+ return VideoBackend._resolve_references(
+ detect_video_family("MiniMaxAI/MiniMax-H3", None),
+ "ref2va",
+ "diffusers",
+ [_png_data_url()],
+ None,
+ audios,
+ "match",
+ 960,
+ 544,
+ decoded_audios = decoded,
+ )
+
+ checked = _refuse_unreadable_audio([first])
+ assert len(calls) == 1
+ reused = resolve([first], checked)
+ assert len(calls) == 1
+ fresh = resolve([first], None)
+ assert len(calls) == 2
+ (reused_wave, reused_rate), (fresh_wave, fresh_rate) = reused.audios[0], fresh.audios[0]
+ assert reused_rate == fresh_rate == 32_000
+ assert reused_wave.dtype == fresh_wave.dtype
+ assert np.array_equal(reused_wave, fresh_wave)
+ # Different inputs than the check saw: decoded afresh, never the stale waveform.
+ other = resolve([second], checked)
+ assert len(calls) == 3
+ assert other.audios[0][0].shape[0] == 64_000
+ assert _refuse_unreadable_audio(None) is None
+
+
+def test_h3_reference_video_frames_are_rgb_without_an_extra_copy(monkeypatch):
+ pytest.importorskip("av")
+ from PIL import Image
+
+ from core.inference import video_minimax_h3 as h3
+
+ converts = []
+ real_convert = Image.Image.convert
+
+ def counting_convert(self, *args, **kwargs):
+ converts.append(self.mode)
+ return real_convert(self, *args, **kwargs)
+
+ monkeypatch.setattr(Image.Image, "convert", counting_convert)
+ from core.inference.video import _decode_b64_media
+
+ blob = _decode_b64_media(_reference_video_data_url(seconds = 3.0, with_audio = False))
+ frames, _, _ = h3.decode_h3_reference_video(blob, decode_audio = False)
+ assert frames and all(frame.mode == "RGB" for frame in frames)
+ assert converts == []
+
+
+def test_h3_staged_reference_frames_match_one_save_per_frame(tmp_path):
+ """Repeated frame objects (24 fps upsampling) and the writer threads leave the files a plain loop writes."""
+ import hashlib
+
+ from PIL import Image
+
+ from core.inference import video_minimax_h3 as h3
+
+ distinct = [
+ Image.new("RGB", (64, 48), (index * 9 % 256, index, 255 - index)) for index in range(7)
+ ]
+ frames = [distinct[index // 2] for index in range(13)] + [distinct[0]]
+ (tmp_path / "head").mkdir()
+ staged = h3.stage_h3_references(
+ h3.MiniMaxH3References(videos = ((frames, None, None),)), tmp_path / "head"
+ )
+ expected = tmp_path / "expected"
+ expected.mkdir()
+ for frame_index, frame in enumerate(frames):
+ frame.save(expected / f"{frame_index:05d}.png", format = "PNG", compress_level = 1)
+
+ directory = tmp_path / "head" / "ref-video-00"
+ assert staged.videos == (str(directory),)
+ names = sorted(path.name for path in directory.iterdir())
+ assert names == sorted(path.name for path in expected.iterdir())
+ for name in names:
+ assert (
+ hashlib.sha256((directory / name).read_bytes()).digest()
+ == hashlib.sha256((expected / name).read_bytes()).digest()
+ )
+
+
+def test_h3_staged_reference_frames_raise_the_first_failure_in_frame_order(tmp_path, monkeypatch):
+ from PIL import Image
+
+ from core.inference import video_minimax_h3 as h3
+
+ frames = [Image.new("RGB", (16, 16), (index, 0, 0)) for index in range(8)]
+ real_save = Image.Image.save
+
+ def failing_save(self, fp, *args, **kwargs):
+ if self.getpixel((0, 0))[0] in (3, 6):
+ raise OSError(f"disk full at {self.getpixel((0, 0))[0]}")
+ return real_save(self, fp, *args, **kwargs)
+
+ monkeypatch.setattr(Image.Image, "save", failing_save)
+ with pytest.raises(OSError, match = "disk full at 3"):
+ h3.stage_h3_references(h3.MiniMaxH3References(videos = ((frames, None, None),)), tmp_path)