Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio video: decode reference audio once per request and stage H3 reference frames faster (#16324)
Daniel Hane023bac16dd8
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 2af396d18dc..1e33e127324 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -11795,6 +11795,7 @@ class VideoBackend:flow_shift: Optional[float] = None,audio_flow_shift: Optional[float] = None,video_id: Optional[str] = None,+ decoded_reference_audios: Optional[Any] = None,expected_state: Optional[object] = None,live_preview: Optional[bool] = None,before_commit: Optional[Callable[[], None]] = None,@@ -11836,6 +11837,7 @@ class VideoBackend:reference_image_size,canvas_w,canvas_h,+ decoded_audios = decoded_reference_audios,)shift, audio_shift = self._resolve_flow_shifts(state.family, state.engine, flow_shift, audio_flow_shift@@ -13159,6 +13161,8 @@ class VideoBackend:reference_image_size: Optional[str],width: int,height: int,+ *,+ decoded_audios: Optional[Any] = None,) -> Any:"""Sizing is settled here so both engines' min(1, ...) scales are no-ops; native refuses max."""from .video_minimax_h3 import (@@ -13231,13 +13235,16 @@ class VideoBackend:"silent earlier video. Reorder the videos so ones with soundtracks ""come first, or add a replacement soundtrack to each earlier video.")- decoded_audios = tuple(- decode_h3_reference_audio(_decode_b64_media(item)) for item in audios- )+ # The route's readability check may have decoded these exact inputs already.+ reused_audios = decoded_audios.for_inputs(audios) if decoded_audios is not None else None+ if reused_audios is None:+ reused_audios = tuple(+ decode_h3_reference_audio(_decode_b64_media(item)) for item in audios+ )return MiniMaxH3References(images = fitted_images,videos = tuple(decoded_videos),- audios = decoded_audios,+ audios = reused_audios,)def _h3_native_server_render(diff --git a/studio/backend/core/inference/video_minimax_h3.py b/studio/backend/core/inference/video_minimax_h3.pyindex 7704b02c3ca..87a81bf94c4 100644--- a/studio/backend/core/inference/video_minimax_h3.py+++ b/studio/backend/core/inference/video_minimax_h3.py@@ -605,7 +605,9 @@ def _fit_h3_video_frame() -> tuple[Any, tuple[int, int]]:from PIL import Image- image = frame.to_image().convert("RGB")+ image = frame.to_image()+ if image.mode != "RGB": # PyAV already returns a fresh RGB image; convert only copies it+ image = image.convert("RGB")if fitted_size is None:fitted_size = h3_reference_frame_size(*image.size)if image.size != fitted_size:@@ -974,6 +976,17 @@ class MiniMaxH3References:return len(self.images) + len(self.videos) + len(self.audios)+@dataclass(frozen = True)+class DecodedReferenceAudios:+ """Untrimmed ``decode_h3_reference_audio`` results for ``sources``, reused only for those exact inputs."""++ sources: tuple = ()+ decoded: tuple = ()++ def for_inputs(self, audios: list) -> Optional[tuple]:+ return self.decoded if tuple(audios) == self.sources else None++@dataclass(frozen = True)class MiniMaxH3StagedReferences:"""``MiniMaxH3References`` written to disk the way sd-cli reads them back."""@@ -985,6 +998,46 @@ class MiniMaxH3StagedReferences:audios: tuple[str, ...] = ()+# Pillow's zlib encoder releases the GIL, so a few threads write a reference video's PNGs in parallel.+H3_REF_FRAME_WRITERS = 4+++def _write_h3_reference_frames(directory: Path, frames: Any) -> None:+ """Write ``frames`` as 00000.png onward, encoding a repeated frame object (24 fps fill) once."""+ import io+ from concurrent.futures import ThreadPoolExecutor++ # Keyed by identity: ``frames`` holds every object, so no id is reused while this runs.+ slots: dict[int, tuple[Any, list[Path]]] = {}+ for frame_index, frame in enumerate(frames):+ slots.setdefault(id(frame), (frame, []))[1].append(directory / f"{frame_index:05d}.png")++ def write(slot: tuple[Any, list[Path]]) -> None:+ frame, paths = slot+ if len(paths) == 1:+ frame.save(paths[0], format = "PNG", compress_level = 1)+ return+ buffer = io.BytesIO()+ frame.save(buffer, format = "PNG", compress_level = 1)+ data = buffer.getvalue()+ for path in paths:+ path.write_bytes(data)++ work = list(slots.values())+ workers = min(H3_REF_FRAME_WRITERS, len(work), os.cpu_count() or 1)+ if workers <= 1:+ for slot in work:+ write(slot)+ return+ pool = ThreadPoolExecutor(max_workers = workers, thread_name_prefix = "h3-ref-frames")+ try:+ futures = [pool.submit(write, slot) for slot in work]+ for future in futures:+ future.result() # first failure in frame order, as the sequential loop raised+ finally:+ pool.shutdown(wait = True, cancel_futures = True)++def stage_h3_references(references: MiniMaxH3References, scratch: Path) -> MiniMaxH3StagedReferences:@@ -1002,8 +1055,7 @@ def stage_h3_references(for index, (frames, waveform, sample_rate) in enumerate(references.videos):directory = scratch / f"ref-video-{index:02d}"directory.mkdir()- for frame_index, frame in enumerate(frames):- frame.save(directory / f"{frame_index:05d}.png", format = "PNG", compress_level = 1)+ _write_h3_reference_frames(directory, frames)videos.append(str(directory))if waveform is None:# Positional pairing cannot express "skip this one", so stop pairing here.diff --git a/studio/backend/routes/video.py b/studio/backend/routes/video.pyindex b025223b26b..a9ba657e2ed 100644--- a/studio/backend/routes/video.py+++ b/studio/backend/routes/video.py@@ -1942,25 +1942,32 @@ def _audio_ref_data_url(ref) -> str:)-def _refuse_unreadable_audio(audios: Optional[list[str]]) -> None:- """Decode each reference audio the engine would, refusing one it could not read."""+def _refuse_unreadable_audio(audios: Optional[list[str]]):+ """Decode each reference audio the engine would, refusing one it could not read.++ Returns the decoded audios for this request's render to reuse, or None when nothing was decoded."""if not audios:- return+ return Nonefrom core.inference.video import _decode_b64_mediatry:- from core.inference.video_minimax_h3 import decode_h3_reference_audio+ from core.inference.video_minimax_h3 import (+ DecodedReferenceAudios,+ decode_h3_reference_audio,+ )except ImportError: # no media decoder here; begin_generate reports it- return+ return None+ decoded = []for index, item in enumerate(audios):try:- decode_h3_reference_audio(_decode_b64_media(item))+ decoded.append(decode_h3_reference_audio(_decode_b64_media(item)))except ImportError:- return+ return Noneexcept Exception: # noqa: BLE001 -- any decode failure is client input feedbackraise _openai_video_error(400, f"reference_audios[{index}] is not readable audio.", param = "reference_audios")+ return DecodedReferenceAudios(sources = tuple(audios), decoded = tuple(decoded))def _conditioning_param(@@ -2264,6 +2271,7 @@ async def _start_openai_video(size, seconds, reference_audios = inputs.size, inputs.seconds, inputs.reference_audioswidth, height = size if size is not None else (None, None)audio_checked = [False]+ decoded_audios = [None]def _refuse_unservable_request(pick) -> None:from core.inference.media_model_index import expected_partition@@ -2274,7 +2282,7 @@ async def _start_openai_video(return_refuse_family_inputs(fam, expected_partition(pick), body, reference, inputs)# Decoded here, like the images above, so unreadable audio never costs a switch.- _refuse_unreadable_audio(reference_audios)+ decoded_audios[0] = _refuse_unreadable_audio(reference_audios)audio_checked[0] = Truepin_requested_model = bool(@@ -2297,7 +2305,7 @@ async def _start_openai_video()if not audio_checked[0]:# No switch ran the check, and the engine's own decode can fail with more than ValueError.- await asyncio.to_thread(_refuse_unreadable_audio, reference_audios)+ decoded_audios[0] = await asyncio.to_thread(_refuse_unreadable_audio, reference_audios)backend = get_video_backend()# Managed callers are authorized against the exact resident state, pinned to the reservation;@@ -2353,6 +2361,8 @@ async def _start_openai_video(reference_audios = reference_audios,)generate_kwargs.update({k: v for k, v in controls.items() if v is not None})+ if decoded_audios[0] is not None:+ generate_kwargs["decoded_reference_audios"] = decoded_audios[0]if expected_state is not None:generate_kwargs["expected_state"] = expected_stateresolved = await asyncio.to_thread(diff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.pyindex 5c5bda27d4d..45ba36086f8 100644--- a/studio/backend/tests/test_video_backend.py+++ b/studio/backend/tests/test_video_backend.py@@ -13938,3 +13938,138 @@ def test_h3_unified_guard_reruns_before_a_failed_hosted_encoder_goes_dense(assert free_mib < initial_memory.free_mibassert snapshots == [initial_memory]assert calls == [("fp8", "int8", initial_memory), ("fp8", None, initial_memory)]+++def _png_data_url(width = 64, height = 64):+ import base64+ import io++ from PIL import Image++ buffer = io.BytesIO()+ Image.new("RGB", (width, height), (10, 20, 30)).save(buffer, "PNG")+ return "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode("ascii")+++def test_h3_render_reuses_the_audio_the_route_already_decoded(monkeypatch):+ """The readability check's decode is handed to the render only for the same inputs."""+ pytest.importorskip("av")+ import numpy as np++ from core.inference import video_minimax_h3 as h3+ from core.inference.video import VideoBackend+ from core.inference.video_families import detect_video_family+ from routes.video import _refuse_unreadable_audio++ first = _data_url_wav(seconds = 1.0, rate = 32_000)+ second = _data_url_wav(seconds = 2.0, rate = 32_000)+ real_decode = h3.decode_h3_reference_audio+ calls = []++ def counting_decode(*args, **kwargs):+ calls.append(1)+ return real_decode(*args, **kwargs)++ monkeypatch.setattr(h3, "decode_h3_reference_audio", counting_decode)++ def resolve(audios, decoded):+ return VideoBackend._resolve_references(+ detect_video_family("MiniMaxAI/MiniMax-H3", None),+ "ref2va",+ "diffusers",+ [_png_data_url()],+ None,+ audios,+ "match",+ 960,+ 544,+ decoded_audios = decoded,+ )++ checked = _refuse_unreadable_audio([first])+ assert len(calls) == 1+ reused = resolve([first], checked)+ assert len(calls) == 1+ fresh = resolve([first], None)+ assert len(calls) == 2+ (reused_wave, reused_rate), (fresh_wave, fresh_rate) = reused.audios[0], fresh.audios[0]+ assert reused_rate == fresh_rate == 32_000+ assert reused_wave.dtype == fresh_wave.dtype+ assert np.array_equal(reused_wave, fresh_wave)+ # Different inputs than the check saw: decoded afresh, never the stale waveform.+ other = resolve([second], checked)+ assert len(calls) == 3+ assert other.audios[0][0].shape[0] == 64_000+ assert _refuse_unreadable_audio(None) is None+++def test_h3_reference_video_frames_are_rgb_without_an_extra_copy(monkeypatch):+ pytest.importorskip("av")+ from PIL import Image++ from core.inference import video_minimax_h3 as h3++ converts = []+ real_convert = Image.Image.convert++ def counting_convert(self, *args, **kwargs):+ converts.append(self.mode)+ return real_convert(self, *args, **kwargs)++ monkeypatch.setattr(Image.Image, "convert", counting_convert)+ from core.inference.video import _decode_b64_media++ blob = _decode_b64_media(_reference_video_data_url(seconds = 3.0, with_audio = False))+ frames, _, _ = h3.decode_h3_reference_video(blob, decode_audio = False)+ assert frames and all(frame.mode == "RGB" for frame in frames)+ assert converts == []+++def test_h3_staged_reference_frames_match_one_save_per_frame(tmp_path):+ """Repeated frame objects (24 fps upsampling) and the writer threads leave the files a plain loop writes."""+ import hashlib++ from PIL import Image++ from core.inference import video_minimax_h3 as h3++ distinct = [+ Image.new("RGB", (64, 48), (index * 9 % 256, index, 255 - index)) for index in range(7)+ ]+ frames = [distinct[index // 2] for index in range(13)] + [distinct[0]]+ (tmp_path / "head").mkdir()+ staged = h3.stage_h3_references(+ h3.MiniMaxH3References(videos = ((frames, None, None),)), tmp_path / "head"+ )+ expected = tmp_path / "expected"+ expected.mkdir()+ for frame_index, frame in enumerate(frames):+ frame.save(expected / f"{frame_index:05d}.png", format = "PNG", compress_level = 1)++ directory = tmp_path / "head" / "ref-video-00"+ assert staged.videos == (str(directory),)+ names = sorted(path.name for path in directory.iterdir())+ assert names == sorted(path.name for path in expected.iterdir())+ for name in names:+ assert (+ hashlib.sha256((directory / name).read_bytes()).digest()+ == hashlib.sha256((expected / name).read_bytes()).digest()+ )+++def test_h3_staged_reference_frames_raise_the_first_failure_in_frame_order(tmp_path, monkeypatch):+ from PIL import Image++ from core.inference import video_minimax_h3 as h3++ frames = [Image.new("RGB", (16, 16), (index, 0, 0)) for index in range(8)]+ real_save = Image.Image.save++ def failing_save(self, fp, *args, **kwargs):+ if self.getpixel((0, 0))[0] in (3, 6):+ raise OSError(f"disk full at {self.getpixel((0, 0))[0]}")+ return real_save(self, fp, *args, **kwargs)++ monkeypatch.setattr(Image.Image, "save", failing_save)+ with pytest.raises(OSError, match = "disk full at 3"):+ h3.stage_h3_references(h3.MiniMaxH3References(videos = ((frames, None, None),)), tmp_path)