Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep MiniMax-H3 GGUF renders on a resident sd-server, released when idle (#12461)
Daniel Hanc29edef883dd
diff --git a/studio/backend/core/export/orchestrator.py b/studio/backend/core/export/orchestrator.pyindex f7d5dc15ef2..c712f47f8d8 100644--- a/studio/backend/core/export/orchestrator.py+++ b/studio/backend/core/export/orchestrator.py@@ -207,6 +207,12 @@ class ExportOrchestrator:return Truedef _spawn_subprocess(self, config: dict) -> None:+ # Export does not evict loaded models; at least free an idle resident H3 sd-server.+ try:+ from core.inference.video_minimax_h3 import release_h3_native_servers+ release_h3_native_servers("export subprocess starting")+ except Exception as exc: # noqa: BLE001 - never block an export on this+ logger.warning("Could not release the idle video sd-server for export: %s", exc)# Inside an op a reservation is an install about to abort on is_export_active(), so raising here# would kill the export for an install that never proceeds.from utils.transformers_version import sidecar_swap_in_progressdiff --git a/studio/backend/core/inference/gpu_arbiter.py b/studio/backend/core/inference/gpu_arbiter.pyindex cd5392c5054..f829ed6a206 100644--- a/studio/backend/core/inference/gpu_arbiter.py+++ b/studio/backend/core/inference/gpu_arbiter.py@@ -72,6 +72,14 @@ def _evict_diffusion() -> None:get_active_diffusion_engine().unload()+def _release_idle_video_servers(reason: str) -> None:+ try:+ from core.inference.video_minimax_h3 import release_h3_native_servers+ release_h3_native_servers(reason)+ except Exception as exc: # noqa: BLE001 - never block an acquire on this+ logger.warning("gpu_arbiter: could not release the idle video sd-server: %s", exc)++def _evict_video() -> None:from core.inference.video import get_video_backendget_video_backend().unload()@@ -192,6 +200,9 @@ def acquire_for(raise GpuBusyForAnotherAccountError(_owner, busy)logger.info("gpu_arbiter: evicting %s for %s", _owner, owner)_EVICTORS[_owner]()+ if owner != VIDEO:+ # A resident H3 sd-server sits outside every owner's teardown.+ _release_idle_video_servers(f"GPU acquired for {owner}")# Records who LOADED the model; a plain re-assert must not hand it to whoever asked last.claims = _owner != owner or register is not None or replacing_owner = ownerdiff --git a/studio/backend/core/inference/sd_cpp_args.py b/studio/backend/core/inference/sd_cpp_args.pyindex fef1045cb5e..ce8979c81ea 100644--- a/studio/backend/core/inference/sd_cpp_args.py+++ b/studio/backend/core/inference/sd_cpp_args.py@@ -552,6 +552,8 @@ def build_sd_cpp_server_command(cmd: list[str] = [binary, "--diffusion-model", files.diffusion_model]for flag, value in (("--vae", files.vae),+ # Without it a resident H3 vid_gen server renders a silent clip.+ ("--audio-vae", files.audio_vae),("--clip_l", files.clip_l),("--clip_g", files.clip_g),("--t5xxl", files.t5xxl),@@ -653,6 +655,59 @@ def build_img_gen_request(return req+def h3_server_eligible(params: "SdCppVideoGenParams") -> bool:+ """False for reference videos / audio: path-only sd-cli flags with no ``vid_gen`` JSON field."""+ return not (params.ref_videos or params.ref_video_audios or params.ref_audios)+++def build_vid_gen_request(+ params: "SdCppVideoGenParams",+ *,+ images_b64: Optional[dict[str, str]] = None,+ ref_images_b64: Optional[list[str]] = None,+ output_compression: int = 90,+) -> dict:+ """``POST /sdcpp/v1/vid_gen`` body equivalent to ``build_sd_cpp_video_command`` (same pixels for the same seed).++ AVI: needs no WebM build, as the CUDA prebuilt's sd-cli writes. ``--rng cpu`` is a server context flag, passed at+ spawn."""+ if not (params.prompt or "").strip():+ raise ValueError("prompt is required")+ if params.width <= 0 or params.height <= 0 or params.num_frames <= 0:+ raise ValueError("width, height, and num_frames must be positive")+ if not h3_server_eligible(params):+ raise ValueError("reference videos and audio are not accepted by the sd-server vid_gen API")+ images = dict(images_b64 or {})+ if (images.get("init_image") or images.get("end_image")) and ref_images_b64:+ raise ValueError(+ "MiniMax-H3 keyframes and references cannot be combined: they run against "+ "different denoiser partitions."+ )+ sample_params: dict = {"guidance": {"txt_cfg": float(params.cfg_scale)}}+ if params.steps is not None:+ sample_params["sample_steps"] = int(params.steps)+ if params.flow_shift is not None:+ sample_params["flow_shift"] = float(params.flow_shift)+ req: dict = {+ "prompt": params.prompt,+ "width": int(params.width),+ "height": int(params.height),+ "video_frames": int(params.num_frames),+ "fps": int(params.fps),+ "sample_params": sample_params,+ "output_format": "avi",+ "output_compression": int(output_compression),+ }+ if params.seed is not None:+ req["seed"] = int(params.seed)+ for key in ("init_image", "end_image"):+ if images.get(key):+ req[key] = images[key]+ if ref_images_b64:+ req["ref_images"] = list(ref_images_b64)+ return req++def _fmt_float(value: float) -> str:"""Compact float -> str: drop a trailing ``.0`` (``1.0`` -> ``1``) to keep logged commands tidy."""f = float(value)diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.pyindex df958248c36..e94b068d6ad 100644--- a/studio/backend/core/inference/sd_cpp_backend.py+++ b/studio/backend/core/inference/sd_cpp_backend.py@@ -26,6 +26,7 @@ import reimport sysimport threadingimport time+import weakreffrom dataclasses import dataclass, field, replacefrom pathlib import Pathfrom typing import Any, Optional@@ -1568,7 +1569,33 @@ def _managed_tree_in_use() -> bool:Reads the singleton without a lock on purpose: a stale answer either defers an upgrade to thenext load (harmless) or lets one through in a window the load path guards anyway."""- return _tree_in_use(_sd_cpp_backend)+ return _tree_in_use(_sd_cpp_backend) or _external_tree_holder_alive()+++# Other backends' processes running out of the managed tree (the H3 video sd-server), so an install stands down for+# them too. Weak: a dropped runtime cannot pin the tree.+_external_tree_holders: "weakref.WeakSet[Any]" = weakref.WeakSet()+++def register_tree_holder(holder: Any) -> None:+ with _tree_state:+ _external_tree_holders.add(holder)+++def unregister_tree_holder(holder: Any) -> None:+ with _tree_state:+ _external_tree_holders.discard(holder)+ _tree_state.notify_all()+++def _external_tree_holder_alive() -> bool:+ for holder in list(_external_tree_holders):+ try:+ if holder.is_alive():+ return True+ except Exception: # noqa: BLE001 -- a broken holder must not wedge installs either way+ continue+ return Falsedef _accelerator_changed(binary: str, accelerator: str) -> bool:diff --git a/studio/backend/core/inference/sd_cpp_server.py b/studio/backend/core/inference/sd_cpp_server.pyindex 5076c29abba..8d1d23b58f0 100644--- a/studio/backend/core/inference/sd_cpp_server.py+++ b/studio/backend/core/inference/sd_cpp_server.py@@ -15,6 +15,8 @@ Shape mirrors ``core/rag/embed_llama_server.py``:drain stdout on a daemon thread, poll until ready.* ``img_gen`` -- POST ``/sdcpp/v1/img_gen`` (the whole batch in one request),poll the async job to a terminal state, return image bytes.+ * ``vid_gen`` -- POST ``/sdcpp/v1/vid_gen`` (MiniMax-H3 native video), same polling,+ return the encoded container (video + muxed audio).* ``stop`` -- SIGTERM -> wait -> SIGKILL, join the drain thread (idempotent).Readiness is real: upstream ``main.cpp`` loads the model BEFORE it binds the port and@@ -78,6 +80,7 @@ _TRANSPORT_ERRORS = (# capabilities endpoint (which can block)._READY_PATH = "/v1/models"_IMG_GEN_PATH = "/sdcpp/v1/img_gen"+_VID_GEN_PATH = "/sdcpp/v1/vid_gen"_JOBS_PATH = "/sdcpp/v1/jobs"# Stable parameter residency reported once during startup. Compute-buffer lines are deliberately excluded: those@@ -155,6 +158,17 @@ def _has_ancestor(return False+class SdCppServerUnsupported(RuntimeError):+ """The sd-server has no route / mode for this job kind; nothing ran, so the caller can fall back to sd-cli."""+++def _route_unsupported(status_code: int, text: str, kind: str) -> bool:+ # 404: no route in this build; 400 "does not support": the loaded model lacks the mode (runtime.cpp).+ if status_code == 404:+ return True+ return status_code == 400 and f"does not support {kind}" in (text or "")++class SdCppServer:"""A resident ``sd-server`` subprocess plus the HTTP client that drives it."""@@ -525,6 +539,62 @@ class SdCppServer:) -> list[bytes]:"""Submit one async ``img_gen`` job, poll it to completion, return image bytes.+ ``on_step`` receives each server stdout line (for the step bar). ``cancel_event``,+ when set, cancels the job via the native endpoint and raises ``SdCppCancelled``.+ Raises ``RuntimeError`` on submit/poll failures (including the server dying), with+ the log tail attached.+ """+ return self._run_job(+ "img_gen",+ _IMG_GEN_PATH,+ payload,+ self._decode_images,+ on_step = on_step,+ cancel_event = cancel_event,+ poll_interval = poll_interval,+ submit_timeout = submit_timeout,+ total_timeout = total_timeout,+ )++ def vid_gen(+ self,+ payload: dict[str, Any],+ *,+ on_step: Optional[Callable[[str], None]] = None,+ cancel_event: Optional[threading.Event] = None,+ poll_interval: float = 0.4,+ submit_timeout: float = 60.0,+ total_timeout: float = NATIVE_GENERATION_TIMEOUT_S,+ ) -> bytes:+ """``img_gen``'s contract for one ``vid_gen`` job, returning the container bytes (audio muxed); raises+ ``SdCppServerUnsupported`` when the server cannot do video."""+ return self._run_job(+ "vid_gen",+ _VID_GEN_PATH,+ payload,+ self._decode_video,+ on_step = on_step,+ cancel_event = cancel_event,+ poll_interval = poll_interval,+ submit_timeout = submit_timeout,+ total_timeout = total_timeout,+ )++ def _run_job(+ self,+ kind: str,+ path: str,+ payload: dict[str, Any],+ decode: Callable[[dict[str, Any]], Any],+ *,+ on_step: Optional[Callable[[str], None]] = None,+ cancel_event: Optional[threading.Event] = None,+ poll_interval: float = 0.4,+ submit_timeout: float = 60.0,+ total_timeout: float = NATIVE_GENERATION_TIMEOUT_S,+ ) -> Any:+ """Submit one async ``kind`` job at ``path``, poll it to completion, return ``decode(job)``.+``on_step`` receives each server stdout line (for the step bar). ``cancel_event``,when set, cancels the job via the native endpoint and raises ``SdCppCancelled``.Raises ``RuntimeError`` on submit/poll failures (including the server dying), with@@ -541,29 +611,34 @@ class SdCppServer:try:try:resp = self._client.post(- f"{self.base_url}{_IMG_GEN_PATH}", json = payload, timeout = submit_timeout+ f"{self.base_url}{path}", json = payload, timeout = submit_timeout)except (*_TRANSPORT_ERRORS, httpx.TimeoutException) as exc:- raise RuntimeError(self._died_message("img_gen submit", exc)) from exc+ raise RuntimeError(self._died_message(f"{kind} submit", exc)) from excif resp.status_code == 429:raise RuntimeError("sd-server job queue is full (HTTP 429).")if resp.status_code not in (200, 202):+ if _route_unsupported(resp.status_code, resp.text, kind):+ raise SdCppServerUnsupported(+ f"this sd-server does not serve {kind} (HTTP {resp.status_code}): "+ f"{resp.text[:300]}"+ )raise RuntimeError(- f"sd-server img_gen submit -> {resp.status_code}: {resp.text[:500]}"+ f"sd-server {kind} submit -> {resp.status_code}: {resp.text[:500]}")try:job = resp.json()except ValueError as exc:raise RuntimeError(- f"sd-server img_gen returned a non-JSON submit response: {exc}"+ f"sd-server {kind} returned a non-JSON submit response: {exc}") from excif not isinstance(job, dict):raise RuntimeError(- f"sd-server img_gen returned an unexpected submit response type: {type(job)}"+ f"sd-server {kind} returned an unexpected submit response type: {type(job)}")job_id = job.get("id")if not job_id:- raise RuntimeError(f"sd-server img_gen returned no job id: {job}")+ raise RuntimeError(f"sd-server {kind} returned no job id: {job}")deadline = time.monotonic() + total_timeoutcancel_sent_at: Optional[float] = None@@ -583,7 +658,7 @@ class SdCppServer:# Unwinding a cancel (e.g. unload killed the server): clean cancellation.if cancel_event is not None and cancel_event.is_set():raise SdCppCancelled("sd-server generation was cancelled.")- raise RuntimeError(self._died_message("img_gen poll", None))+ raise RuntimeError(self._died_message(f"{kind} poll", None))if time.monotonic() > deadline:# sd-server will not interrupt an in-flight job, so cancel + stop to free the slot; the backend# reloads on the next generate.@@ -616,13 +691,19 @@ class SdCppServer:)status = jd.get("status")if status == _TERMINAL_OK:- return self._decode_images(jd)+ return decode(jd)if status == _TERMINAL_FAIL:err = jd.get("error") or {}- raise RuntimeError(+ message = ("sd-server generation failed: "f"{err.get('code', 'error')}: {err.get('message', '')}".strip())+ if kind == "vid_gen":+ # The job error is generic; the cause (OOM, abort) is only in the log.+ tail = _diagnostic_tail(self._tail)+ if tail:+ message += "\nLast output:\n" + tail[:1500]+ raise RuntimeError(message)if status == _TERMINAL_CANCELLED:raise SdCppCancelled("sd-server generation was cancelled.")time.sleep(poll_interval)@@ -636,6 +717,17 @@ class SdCppServer:except Exception: # noqa: BLE001 -- cancel is best-effortpass+ @staticmethod+ def _decode_video(job: dict[str, Any]) -> bytes:+ result = job.get("result") if isinstance(job, dict) else None+ b64 = result.get("b64_json") if isinstance(result, dict) else None+ if not b64:+ raise RuntimeError("sd-server completed the job but returned no video.")+ try:+ return base64.b64decode(b64)+ except Exception as exc: # noqa: BLE001+ raise RuntimeError(f"sd-server returned an undecodable video: {exc}") from exc+@staticmethoddef _decode_images(job: dict[str, Any]) -> list[bytes]:# Type-check each level before indexing (guards a misbehaving/older server).diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex d9debeeac88..44b1ec385a2 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -42,7 +42,7 @@ import timeimport typesfrom dataclasses import dataclassfrom pathlib import Path-from typing import Any, Optional+from typing import Any, Callable, Optionalfrom hub.utils.hf_errors import modelscope_missingfrom loggers import get_logger@@ -1290,10 +1290,13 @@ def _h3_free_device_bytes(device: str) -> Optional[int]:return None-def _h3_card_free_bytes(device: Optional[str], ordinal: Optional[int]) -> Optional[int]:- """Live free VRAM via nvidia-smi / amd-smi (no CUDA context here), or None. No pin = sd.cpp's ordinal 0."""+def _h3_card_memory_bytes(+ device: Optional[str], ordinal: Optional[int]+) -> tuple[Optional[int], Optional[int]]:+ """(free, total) VRAM of the H3 render card via nvidia-smi / amd-smi (no CUDA context), or (None, None). No pin =+ ordinal 0."""if device != "cuda":- return None+ return None, Nonetry:from utils.hardware import get_visible_gpu_utilization, gpu_query@@ -1306,11 +1309,44 @@ def _h3_card_free_bytes(device: Optional[str], ordinal: Optional[int]) -> Optiontotal = entry.get("vram_total_gb")used = entry.get("vram_used_gb")if total is None or used is None:- return None- return max(0, int((float(total) - float(used)) * 1024**3))+ return None, None+ total_bytes = int(float(total) * 1024**3)+ return max(0, int((float(total) - float(used)) * 1024**3)), total_bytesexcept Exception: # noqa: BLE001 -- an unreadable card decides nothing- return None- return None+ return None, None+ return None, None+++def _h3_card_free_bytes(device: Optional[str], ordinal: Optional[int]) -> Optional[int]:+ """Free VRAM of the H3 render card, or None (keeps the committed offload flags)."""+ return _h3_card_memory_bytes(device, ordinal)[0]+++def _h3_native_server_pressure(device: Optional[str], ordinal: Optional[int]) -> Optional[str]:+ """Why the resident H3 sd-server should not outlive a render (``h3_native_server_pressure``), or None."""+ from .diffusion_memory import (+ _available_system_memory_mib,+ _cgroup_memory_limit_mib,+ _system_memory_mib,+ )+ from .video_minimax_h3 import h3_native_server_pressure++ vram_free, vram_total = _h3_card_memory_bytes(device, ordinal)+ try:+ host_total_mib = _system_memory_mib()[0]+ # Available RAM is cgroup-capped, so the reserve must be a share of the same limit.+ cgroup_limit_mib = _cgroup_memory_limit_mib()+ if cgroup_limit_mib and (host_total_mib is None or cgroup_limit_mib < host_total_mib):+ host_total_mib = cgroup_limit_mib+ host_avail_mib = _available_system_memory_mib()+ except Exception: # noqa: BLE001 -- unknown host memory decides nothing+ host_total_mib = host_avail_mib = None+ return h3_native_server_pressure(+ vram_free = vram_free,+ vram_total = vram_total,+ host_available = None if host_avail_mib is None else int(host_avail_mib) << 20,+ host_total = None if host_total_mib is None else int(host_total_mib) << 20,+ )def _h3_device_capacity_bytes(device: str) -> Optional[int]:@@ -3533,22 +3569,41 @@ class VideoBackend:# After the policy, so the pin can see which modules it left on the CPU; without it sd.cpp uses ordinal 0# whatever was selected.native_offload += tuple(device_backend_flags(native_device_name, list(native_offload)))- from .video_minimax_h3 import MiniMaxH3NativeRuntime+ from .video_minimax_h3 import (+ H3NativeServerSlot,+ MiniMaxH3NativeRuntime,+ h3_sibling_server_binary,+ )+ native_files = SdCppModelFiles(+ diffusion_model = str(resolved[0]),+ llm = str(resolved[1]),+ vae = str(resolved[2]),+ audio_vae = str(resolved[3]),+ )+ # Nothing spawns here: the slot starts the server on the first render; no sibling sd-server keeps one-shot.+ server_binary = h3_sibling_server_binary(getattr(engine, "binary", None))runtime = MiniMaxH3NativeRuntime(engine = engine,# Pinned under the reader claim above, where this exact file answered --help with the H3 options. Taking it# at generation time instead would compare a replacement against itself.binary_identity = binary_identity,selected_card = selected_card,- files = SdCppModelFiles(- diffusion_model = str(resolved[0]),- llm = str(resolved[1]),- vae = str(resolved[2]),- audio_vae = str(resolved[3]),- ),+ files = native_files,offload_flags = native_offload,env = native_env,+ server_slot = (+ H3NativeServerSlot(+ server_binary,+ native_files,+ native_offload,+ pressure_probe = functools.partial(+ _h3_native_server_pressure, native_device, native_ordinal+ ),+ )+ if server_binary+ else None+ ),)with self._lock:@@ -8835,6 +8890,97 @@ class VideoBackend:audios = decoded_audios,)+ def _h3_native_server_render(+ self,+ runtime: Any,+ params: Any,+ *,+ output_path: Path,+ on_log: Callable[[str], None],+ cancel: threading.Event,+ flags: Optional[list[str]] = None,+ env: Optional[dict[str, str]] = None,+ ) -> Optional[Path]:+ """Render one MiniMax-H3 clip on the runtime's resident sd-server, spawned with this render's sd-cli ``flags`` /+ ``env``.++ Returns the written container, or None for the one-shot sd-cli: no slot, kill switch, reference video / audio,+ or a server that cannot start, lacks vid_gen, or died (the runtime then stays one-shot). Every fallback first+ stops the server so the two never hold the model at once. A job the live server fails is raised after stopping+ it. Called inside the generation's managed-tree reader claim."""+ import base64++ from .sd_cpp_args import build_vid_gen_request, h3_server_eligible+ from .sd_cpp_engine import SdCppCancelled+ from .sd_cpp_server import SdCppServerUnsupported+ from .video_minimax_h3 import h3_native_server_enabled++ slot = getattr(runtime, "server_slot", None)+ if slot is None or slot.disabled_reason is not None:+ return None+ if not h3_native_server_enabled():+ slot.stop("UNSLOTH_H3_NATIVE_SERVER=0")+ return None+ if not h3_server_eligible(params):+ # The sd-cli loads the whole model again; a resident copy beside it may not fit.+ slot.release("reference video / audio render runs on sd-cli")+ return None++ def b64_file(path: Optional[str]) -> Optional[str]:+ if not path:+ return None+ return base64.b64encode(Path(path).read_bytes()).decode("ascii")++ payload = build_vid_gen_request(+ params,+ images_b64 = {+ "init_image": b64_file(params.init_img),+ "end_image": b64_file(params.end_img),+ },+ ref_images_b64 = [b64_file(p) for p in params.ref_images] or None,+ )+ render_flags = list(runtime.offload_flags) if flags is None else list(flags)+ render_env = dict(runtime.env) if env is None else dict(env)+ slot.begin_render()+ try:+ try:+ server = slot.get(render_flags, render_env, cancel_event = cancel)+ except SdCppCancelled:+ raise+ except Exception as exc: # noqa: BLE001 - any start failure leaves the one-shot path+ if cancel.is_set():+ raise SdCppCancelled("sd-server start was cancelled.") from exc+ logger.warning("video.h3_native_server_start_failed (using sd-cli): %s", exc)+ slot.disable(f"start failed: {exc}")+ return None+ try:+ data = server.vid_gen(payload, on_step = on_log, cancel_event = cancel)+ except SdCppServerUnsupported as exc:+ logger.warning("video.h3_native_server_unsupported (using sd-cli): %s", exc)+ slot.disable(str(exc))+ return None+ except SdCppCancelled:+ raise+ except RuntimeError as exc:+ if cancel.is_set():+ raise SdCppCancelled("sd-server generation was cancelled.") from exc+ if not server.is_alive():+ logger.warning(+ "video.h3_native_server_died (retrying once on sd-cli, staying one-shot): %s",+ exc,+ )+ slot.disable(f"server died: {exc}")+ return None+ slot.stop("render failed")+ raise+ Path(output_path).write_bytes(data)+ return Path(output_path)+ finally:+ try:+ slot.end_render()+ except Exception as exc: # noqa: BLE001 - the render's own outcome wins+ logger.warning("video.h3_native_server_release_failed: %s", exc)+def _generate_h3_native(self,*,@@ -8882,10 +9028,30 @@ class VideoBackend:need = h3_native_resident_bytes(file_bytes, width, height, frames)except Exception: # noqa: BLE001 -- unknown size keeps the committed offloadneed = None+ free = None+ if need:+ # A live resident server's weights count as free; a larger clip's activations must still fit the rest.+ slot = getattr(runtime, "server_slot", None)+ live = slot.alive_signature() if slot is not None else None+ resident_flags, _ = h3_native_render_flags(+ runtime.offload_flags,+ memory_mode = state.memory_mode,+ free_bytes = need,+ need_bytes = need,+ )+ if (+ live is not None+ and live[2] == tuple(resident_flags)+ and "--offload-to-cpu" not in live[2]+ ):+ card_free = _h3_card_free_bytes(state.device, state.gpu_ordinal)+ free = None if card_free is None else card_free + file_bytes+ else:+ free = _h3_card_free_bytes(state.device, state.gpu_ordinal)render_flags, render_resident = h3_native_render_flags(runtime.offload_flags,memory_mode = state.memory_mode,- free_bytes = _h3_card_free_bytes(state.device, state.gpu_ordinal) if need else None,+ free_bytes = free,need_bytes = need,)if seed is None:@@ -8978,31 +9144,42 @@ class VideoBackend:"replaced by an install, so it is no longer the build this model ""was checked against. Reload the model and try again.")- generated = runtime.engine.generate_video(- runtime.files,- SdCppVideoGenParams(- prompt = prompt,- width = width,- height = height,- num_frames = frames,- fps = fps,- steps = steps,- cfg_scale = 1.0,- seed = int(seed),- init_img = init_img,- end_img = end_img,- ref_images = staged.images,- ref_videos = staged.videos,- ref_video_audios = staged.video_audios,- ref_audios = staged.audios,- flow_shift = flow_shift,- ),- output_path = str(output_path),- offload = render_flags,- env = dict(runtime.env) or None,+ video_params = SdCppVideoGenParams(+ prompt = prompt,+ width = width,+ height = height,+ num_frames = frames,+ fps = fps,+ steps = steps,+ cfg_scale = 1.0,+ seed = int(seed),+ init_img = init_img,+ end_img = end_img,+ ref_images = staged.images,+ ref_videos = staged.videos,+ ref_video_audios = staged.video_audios,+ ref_audios = staged.audios,+ flow_shift = flow_shift,+ )+ generated = self._h3_native_server_render(+ runtime,+ video_params,+ output_path = output_path,on_log = on_log,- cancel_event = cancel,+ cancel = cancel,+ flags = render_flags,+ env = dict(runtime.env),)+ if generated is None:+ generated = runtime.engine.generate_video(+ runtime.files,+ video_params,+ output_path = str(output_path),+ offload = render_flags,+ env = dict(runtime.env) or None,+ on_log = on_log,+ cancel_event = cancel,+ )except SdCppCancelled:raise RuntimeError(VIDEO_CANCELLED_MSG) from Noneexcept RuntimeError as exc:@@ -9164,6 +9341,12 @@ class VideoBackend:cannot observe a half-torn-down backend."""state, self._state = self._state, Noneif state is not None:+ server_slot = getattr(getattr(state, "pipe", None), "server_slot", None)+ if server_slot is not None:+ try:+ server_slot.stop("unload")+ except Exception as exc: # noqa: BLE001 - teardown is best effort+ logger.warning("video.h3_native_server_stop_failed: %s", exc)# An in-flight background compile still runs the denoiser the teardown frees.if getattr(state, "bg_compile", None) is not None:try:diff --git a/studio/backend/core/inference/video_minimax_h3.py b/studio/backend/core/inference/video_minimax_h3.pyindex c1a7bcf050b..32d8d9ccf68 100644--- a/studio/backend/core/inference/video_minimax_h3.py+++ b/studio/backend/core/inference/video_minimax_h3.py@@ -8,9 +8,10 @@ from __future__ import annotationsimport mathimport osimport tempfile+import weakreffrom dataclasses import dataclassfrom pathlib import Path-from typing import Any, Optional+from typing import Any, Callable, Optional# Must stay equal to the minimax-h3 family's `gguf_repo`. They are the same one-click pick, and main's# test_curated_gguf_repos_are_unsloth_mirrors only checks the family field, so a divergence here would let that test@@ -1326,6 +1327,329 @@ class MiniMaxH3NativeRuntime:# The card the load resolved, kept for failure records: re-resolving at failure time can read None.selected_card: Optional[str] = Noneenv: tuple[tuple[str, str], ...] = ()+ # H3NativeServerSlot, or None for one-shot sd-cli only.+ server_slot: Any = None+++# 0 / false / off: every render on a fresh one-shot sd-cli.+H3_NATIVE_SERVER_ENV = "UNSLOTH_H3_NATIVE_SERVER"+H3_NATIVE_SERVER_IDLE_ENV = "UNSLOTH_H3_NATIVE_SERVER_IDLE_S"+# Covers watching a clip and editing the next prompt; an absent user gets the memory back within 3 minutes.+H3_NATIVE_SERVER_IDLE_DEFAULT_S = 180.0+# Free VRAM and available host RAM must each keep max(4 GiB, 15%) for an idle server to stay; same reserve as+# diffusion_memory's _PIN_RESERVE_*.+H3_NATIVE_SERVER_RESERVE_MIN_BYTES = 4 << 30+H3_NATIVE_SERVER_RESERVE_FRACTION = 0.15+++def h3_native_server_enabled() -> bool:+ value = os.environ.get(H3_NATIVE_SERVER_ENV, "").strip().lower()+ return value not in ("0", "false", "no", "off")+++def h3_native_server_idle_s(env: Optional[dict] = None) -> float:+ """The idle window in seconds; an unreadable or negative value falls back to the default."""+ environ = env if env is not None else os.environ+ raw = str(environ.get(H3_NATIVE_SERVER_IDLE_ENV, "") or "").strip()+ if not raw:+ return H3_NATIVE_SERVER_IDLE_DEFAULT_S+ try:+ value = float(raw)+ except ValueError:+ return H3_NATIVE_SERVER_IDLE_DEFAULT_S+ if value != value or value < 0: # NaN or negative+ return H3_NATIVE_SERVER_IDLE_DEFAULT_S+ return value+++def h3_native_server_reserve_bytes(total_bytes: Optional[int]) -> int:+ total = int(total_bytes or 0)+ return max(H3_NATIVE_SERVER_RESERVE_MIN_BYTES, int(total * H3_NATIVE_SERVER_RESERVE_FRACTION))+++def h3_native_server_pressure(+ *,+ vram_free: Optional[int],+ vram_total: Optional[int],+ host_available: Optional[int],+ host_total: Optional[int],+) -> Optional[str]:+ """Why an idle server must not be kept, or None. An unknown reading decides nothing for its side."""+ if vram_free is not None and vram_total:+ floor = h3_native_server_reserve_bytes(vram_total)+ if int(vram_free) < floor:+ return f"free VRAM {int(vram_free) >> 20} MiB below the {floor >> 20} MiB reserve"+ if host_available is not None and host_total:+ floor = h3_native_server_reserve_bytes(host_total)+ if int(host_available) < floor:+ return f"available host RAM {int(host_available) >> 20} MiB below the {floor >> 20} MiB reserve"+ return None+++def h3_sibling_server_binary(cli_binary: Optional[str]) -> Optional[str]:+ """The ``sd-server`` beside the vetted ``sd-cli``, or None. Not ``find_sd_server_binary``: that may resolve an+ unvetted build from ``SD_SERVER_PATH`` / ``PATH``."""+ if not cli_binary:+ return None+ cli = Path(cli_binary)+ suffix = ".exe" if cli.suffix.lower() == ".exe" else ""+ candidate = cli.with_name("sd-server" + suffix)+ return str(candidate) if candidate.is_file() else None+++_LIVE_SLOTS: "weakref.WeakSet[H3NativeServerSlot]" = weakref.WeakSet()+++def release_h3_native_servers(reason: str) -> int:+ """Stop every resident H3 sd-server (a busy one after its render). Returns how many; never raises."""+ count = 0+ for slot in list(_LIVE_SLOTS):+ try:+ if slot.release(reason):+ count += 1+ except Exception: # noqa: BLE001 -- a release must never break the caller that needs the memory+ continue+ return count+++class H3NativeServerSlot:+ """At most one resident ``sd-server`` for a loaded MiniMax-H3 native runtime, so renders skip the ~32 GB reload.++ Spawned with exactly the flags and env the one-shot sd-cli would get; another signature respawns it. Stopped by+ the idle timer, ``pressure_probe`` after a render, ``release`` (busy: at render end) and ``stop`` on unload.+ Registered as a managed-tree holder while alive so an install stands down.+ """++ def __init__(+ self,+ server_binary: str,+ files: Any,+ offload_flags: "tuple[str, ...] | list[str]" = (),+ *,+ pressure_probe: Optional[Callable[[], Optional[str]]] = None,+ ) -> None:+ import threading++ self.server_binary = server_binary+ self.files = files+ self.offload_flags = tuple(offload_flags)+ self.pressure_probe = pressure_probe+ self.disabled_reason: Optional[str] = None+ self.last_release_reason: Optional[str] = None+ self._server: Any = None+ self._stopping: Any = None+ self._signature: Optional[tuple] = None+ self._lock = threading.RLock()+ self._busy = 0+ self._release_pending: Optional[str] = None+ self._timer: Any = None+ # Bumped on every arm / cancel so a timer that fired as it was cancelled sees it is stale.+ self._timer_token = 0+ _LIVE_SLOTS.add(self)++ def is_alive(self) -> bool:+ # A server mid-stop still runs out of the managed tree.+ return any(s is not None and s.is_alive() for s in (self._server, self._stopping))++ @property+ def signature(self) -> Optional[tuple]:+ return self._signature++ def _signature_for(self, flags: "tuple[str, ...]", env: "tuple[tuple[str, str], ...]") -> tuple:+ files = self.files+ file_key = tuple(+ getattr(files, name, None)+ for name in ("diffusion_model", "llm", "vae", "audio_vae", "llm_vision")+ )+ return (self.server_binary, file_key, tuple(flags), tuple(sorted(env)))++ def alive_signature(self) -> Optional[tuple]:+ with self._lock:+ return self._signature if self.is_alive() else None++ def get(+ self,+ flags: "Optional[tuple[str, ...] | list[str]]" = None,+ env: "Optional[dict[str, str] | tuple[tuple[str, str], ...]]" = None,+ *,+ cancel_event: Any = None,+ ) -> Any:+ """The live server for this (flags, env), (re)spawning one if needed. ``cancel_event`` aborts a start: the+ render holds the generate lock through the model load, so unload and Cancel would otherwise wait it out."""+ import threading++ from .sd_cpp_backend import register_tree_holder, unregister_tree_holder+ from .sd_cpp_engine import SdCppCancelled, is_managed_binary+ from .sd_cpp_server import SdCppServer++ flag_tuple = tuple(self.offload_flags if flags is None else flags)+ env_pairs = tuple((env or {}).items()) if isinstance(env, dict) else tuple(env or ())+ wanted = self._signature_for(flag_tuple, env_pairs)+ with self._lock:+ self._cancel_timer_locked()+ if self._server is not None and self._server.is_alive() and self._signature == wanted:+ return self._server+ if self._server is not None:+ self._stop_locked(+ "signature changed" if self._server.is_alive() else "server exited"+ )+ if cancel_event is not None and cancel_event.is_set():+ raise SdCppCancelled("sd-server start was cancelled before launch.")+ server = SdCppServer(self.server_binary)+ managed = is_managed_binary(self.server_binary)+ self._server = server+ self._signature = wanted+ if managed:+ register_tree_holder(self)+ started = threading.Event()+ if cancel_event is not None:++ def abort_on_cancel() -> None:+ # stop() sets the abort before taking the lifecycle lock, so start() leaves its readiness wait.+ while not started.is_set():+ if cancel_event.wait(0.2):+ if not started.is_set():+ server.stop()+ return++ threading.Thread(+ target = abort_on_cancel, daemon = True, name = "h3-sd-server-start-cancel"+ ).start()+ try:+ server.start(+ self.files,+ offload = list(flag_tuple),+ env = dict(env_pairs) or None,+ # A context flag on sd-server (per-run on sd-cli); same RNG keeps seeds identical across paths.+ extra_args = ["--rng", "cpu"],+ )+ except BaseException:+ started.set()+ self._server = None+ self._signature = None+ self._stopping = server+ try:+ server.stop()+ finally:+ self._stopping = None+ unregister_tree_holder(self)+ raise+ started.set()+ if cancel_event is not None and cancel_event.is_set() and not server.is_alive():+ self._stop_locked("cancelled")+ raise SdCppCancelled("sd-server start was cancelled.")+ return server++ def begin_render(self) -> None:+ with self._lock:+ self._busy += 1+ self._cancel_timer_locked()++ def end_render(self) -> Optional[str]:+ """Stop the server (pending release, memory pressure, idle 0) or arm the idle timer. Returns the stop reason."""+ with self._lock:+ self._busy = max(0, self._busy - 1)+ if self._busy or self._server is None:+ return None+ reason = self._release_pending+ if reason is None and self.pressure_probe is not None:+ try:+ reason = self.pressure_probe()+ except Exception: # noqa: BLE001 -- an unreadable probe keeps the idle timeout as the bound+ reason = None+ if reason:+ reason = f"memory pressure: {reason}"+ idle_s = h3_native_server_idle_s()+ if reason is None and idle_s <= 0:+ reason = "idle timeout 0"+ with self._lock:+ if self._busy or self._server is None:+ return None+ if reason is not None:+ self._stop_locked(reason)+ return reason+ self._arm_timer_locked(idle_s)+ return None++ def release(self, reason: str) -> bool:+ """Stop the server for another consumer; a busy one stops at ``end_render``. True when there was one.++ Never waits on a server start (minutes under the lock, and the caller may hold the GPU arbiter): the release is+ left pending for that render instead."""+ if not self._lock.acquire(timeout = 0.5):+ self._release_pending = reason+ return True+ try:+ if self._server is None:+ return False+ if self._busy:+ self._release_pending = reason+ return True+ self._stop_locked(reason)+ return True+ finally:+ self._lock.release()++ def stop(self, reason: str = "stopped") -> None:+ with self._lock:+ self._stop_locked(reason)++ def disable(self, reason: str) -> None:+ self.disabled_reason = reason+ self.stop(reason)++ def _stop_locked(self, reason: str) -> None:+ from .sd_cpp_backend import unregister_tree_holder++ self._cancel_timer_locked()+ server, self._server = self._server, None+ self._signature = None+ self._release_pending = None+ if server is None:+ return+ self._stopping = server+ self.last_release_reason = reason+ try:+ import logging+ logging.getLogger(__name__).info("h3 native sd-server stopped: %s", reason)+ except Exception: # noqa: BLE001+ pass+ try:+ server.stop()+ finally:+ self._stopping = None+ unregister_tree_holder(self)++ def _cancel_timer_locked(self) -> None:+ self._timer_token += 1+ timer, self._timer = self._timer, None+ if timer is not None:+ timer.cancel()++ def _arm_timer_locked(self, idle_s: float) -> None:+ import threading++ self._cancel_timer_locked()+ token = self._timer_token+ ref = weakref.ref(self)++ def fire() -> None:+ slot = ref()+ if slot is not None:+ slot._on_idle(token)++ timer = threading.Timer(idle_s, fire)+ timer.daemon = True+ timer.name = "h3-sd-server-idle"+ self._timer = timer+ timer.start()++ def _on_idle(self, token: int) -> None:+ with self._lock:+ if token != self._timer_token or self._busy:+ return+ self._timer = None+ self._stop_locked("idle timeout")def transcode_video_to_mp4(source: Path, *, fps: int) -> bytes:diff --git a/studio/backend/tests/test_h3_native_server.py b/studio/backend/tests/test_h3_native_server.pynew file mode 100644index 00000000000..958c21ba93f--- /dev/null+++ b/studio/backend/tests/test_h3_native_server.py@@ -0,0 +1,957 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""MiniMax-H3 native renders through a resident sd-server instead of one sd-cli per render.++Hermetic: the server process and its HTTP client are faked; nothing spawns a binary or opens a+socket. The real-binary check (pixels and audio identical to sd-cli for the same seed, keyframe+and text-only) lives with the benchmark evidence, not here."""++from __future__ import annotations++import base64+import sys+import threading+from pathlib import Path++import pytest++from core.inference import sd_cpp_backend+from core.inference import sd_cpp_server as srv+from core.inference.sd_cpp_args import (+ SdCppModelFiles,+ SdCppVideoGenParams,+ build_sd_cpp_server_command,+ build_vid_gen_request,+ h3_server_eligible,+)+from core.inference.sd_cpp_engine import SdCppCancelled+from core.inference.sd_cpp_server import SdCppServerUnsupported+from core.inference.video import VideoBackend+from core.inference import video_minimax_h3 as h3++# Sibling imports need the tests dir on the path: pytest inserts rootdir, not this package.+_TESTS_DIR = str(Path(__file__).resolve().parent)+if _TESTS_DIR not in sys.path:+ sys.path.insert(0, _TESTS_DIR)++from test_sd_cpp_server import _FakeClient, _FakePopen, _Resp, _server_with # noqa: E402++_FILES = SdCppModelFiles(+ diffusion_model = "/m/h3.gguf",+ vae = "/m/vae.safetensors",+ audio_vae = "/m/audio_vae.safetensors",+ llm = "/m/te.gguf",+)+++def _params(**overrides) -> SdCppVideoGenParams:+ values = dict(+ prompt = "a dog on a beach",+ width = 960,+ height = 544,+ num_frames = 124,+ fps = 24,+ steps = 20,+ cfg_scale = 1.0,+ seed = 42,+ flow_shift = 12.0,+ )+ values.update(overrides)+ return SdCppVideoGenParams(**values)+++def patched(monkeypatch):+ monkeypatch.setattr(srv, "adopt_pid", lambda pid: None)+ monkeypatch.setattr(srv, "forget_pid", lambda pid: None)+ monkeypatch.setattr(srv, "child_popen_kwargs", lambda: {})+ monkeypatch.setattr(srv, "windows_hidden_subprocess_kwargs", lambda: {})+ return monkeypatch+++# -- request / argv -----------------------------------------------------------------------------+++def test_vid_gen_request_mirrors_the_cli_argv():+ req = build_vid_gen_request(_params())+ assert req == {+ "prompt": "a dog on a beach",+ "width": 960,+ "height": 544,+ "video_frames": 124,+ "fps": 24,+ "seed": 42,+ "sample_params": {+ "guidance": {"txt_cfg": 1.0},+ "sample_steps": 20,+ "flow_shift": 12.0,+ },+ # AVI: needs no WebM build, and is what the CUDA prebuilt sd-cli writes anyway.+ "output_format": "avi",+ "output_compression": 90,+ }+++def test_vid_gen_request_carries_keyframes_and_rejects_mixing_with_references():+ req = build_vid_gen_request(_params(), images_b64 = {"init_image": "AAA", "end_image": None})+ assert req["init_image"] == "AAA"+ assert "end_image" not in req+ with pytest.raises(ValueError, match = "cannot be combined"):+ build_vid_gen_request(_params(), images_b64 = {"init_image": "AAA"}, ref_images_b64 = ["BBB"])+++def test_reference_video_and_audio_stay_on_the_cli():+ assert h3_server_eligible(_params()) is True+ assert h3_server_eligible(_params(ref_images = ("/r.png",))) is True+ for field in ("ref_videos", "ref_video_audios", "ref_audios"):+ params = _params(**{field: ("/r",)})+ assert h3_server_eligible(params) is False+ with pytest.raises(ValueError):+ build_vid_gen_request(params)+++def test_server_command_passes_the_audio_vae():+ cmd = build_sd_cpp_server_command("/b/sd-server", _FILES, host = "127.0.0.1", port = 1)+ assert cmd[cmd.index("--audio-vae") + 1] == "/m/audio_vae.safetensors"+ no_audio = build_sd_cpp_server_command(+ "/b/sd-server", SdCppModelFiles(diffusion_model = "/m/z.gguf"), host = "h", port = 1+ )+ assert "--audio-vae" not in no_audio+++# -- SdCppServer.vid_gen ------------------------------------------------------------------------+++def test_vid_gen_returns_container_bytes(patched):+ blob = b"RIFF....AVI "+ client = _FakeClient(+ post = lambda url, json: _Resp(202, {"id": "jobV"}),+ get = lambda url: _Resp(+ 200,+ {"status": "completed", "result": {"b64_json": base64.b64encode(blob).decode()}},+ ),+ )+ s = _server_with(_FakePopen(), client)+ assert s.vid_gen({"prompt": "x"}, poll_interval = 0.0) == blob+ assert client.post_calls[0][0].endswith("/sdcpp/v1/vid_gen")+++ "status,text",+ [(404, "Not Found"), (400, '{"error":"loaded model does not support vid_gen"}')],+)+def test_vid_gen_unsupported_is_its_own_error(patched, status, text):+ s = _server_with(_FakePopen(), _FakeClient(post = lambda url, json: _Resp(status, text = text)))+ with pytest.raises(SdCppServerUnsupported):+ s.vid_gen({"prompt": "x"})+++def test_vid_gen_other_400_is_a_plain_failure(patched):+ s = _server_with(+ _FakePopen(), _FakeClient(post = lambda url, json: _Resp(400, text = "invalid parameters"))+ )+ with pytest.raises(RuntimeError) as info:+ s.vid_gen({"prompt": "x"})+ assert not isinstance(info.value, SdCppServerUnsupported)+++def test_vid_gen_failure_carries_the_log_cause(patched):+ s = _server_with(+ _FakePopen(),+ _FakeClient(+ post = lambda url, json: _Resp(202, {"id": "jobF"}),+ get = lambda url: _Resp(+ 200,+ {+ "status": "failed",+ "error": {+ "code": "generation_failed",+ "message": "generate_video returned no results",+ },+ },+ ),+ ),+ )+ s._tail.append(+ "ggml_backend_cuda_buffer_type_alloc_buffer: allocating 5463.50 MiB: out of memory"+ )+ with pytest.raises(RuntimeError, match = "out of memory"):+ s.vid_gen({"prompt": "x"}, poll_interval = 0.0)+++# -- routing in the video backend ---------------------------------------------------------------+++class _FakeServer:+ def __init__(+ self,+ *,+ data = b"AVI",+ exc = None,+ alive_after = True,+ ):+ self.data = data+ self.exc = exc+ self.alive_after = alive_after+ self.payloads = []+ self._alive = True++ def is_alive(self):+ return self._alive++ def vid_gen(+ self,+ payload,+ *,+ on_step = None,+ cancel_event = None,+ ):+ self.payloads.append(payload)+ if on_step is not None:+ on_step("generate_video 960x544x124")+ if self.exc is not None:+ self._alive = self.alive_after+ raise self.exc+ return self.data+++class _FakeSlot:+ def __init__(+ self,+ server = None,+ start_exc = None,+ ):+ self.server = server+ self.start_exc = start_exc+ self.disabled_reason = None+ self.stopped = 0+ self.released = []+ self.got = []+ self.renders = 0+ self.ended = 0++ def get(+ self,+ flags = None,+ env = None,+ *,+ cancel_event = None,+ ):+ self.got.append((flags, env))+ if self.start_exc is not None:+ raise self.start_exc+ return self.server++ def begin_render(self):+ self.renders += 1++ def end_render(self):+ self.ended += 1++ def release(self, reason):+ self.released.append(reason)+ return True++ def alive_signature(self):+ return None++ def stop(self, reason = "stopped"):+ self.stopped += 1++ def disable(self, reason):+ self.disabled_reason = reason+ self.stop()+++class _Runtime:+ def __init__(+ self,+ slot,+ offload_flags = ("--offload-to-cpu",),+ env = (),+ ):+ self.server_slot = slot+ self.offload_flags = offload_flags+ self.env = env+++def _render(+ slot,+ tmp_path,+ params = None,+ cancel = None,+ **kwargs,+):+ return VideoBackend._h3_native_server_render(+ None,+ _Runtime(slot),+ params or _params(),+ output_path = tmp_path / "out.webm",+ on_log = lambda line: None,+ cancel = cancel or threading.Event(),+ **kwargs,+ )+++def test_server_render_writes_the_container(tmp_path, monkeypatch):+ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)+ server = _FakeServer(data = b"AVI-BYTES")+ out = _render(_FakeSlot(server), tmp_path)+ assert out == tmp_path / "out.webm"+ assert out.read_bytes() == b"AVI-BYTES"+ assert server.payloads[0]["seed"] == 42+++def test_keyframes_are_sent_inline(tmp_path, monkeypatch):+ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)+ first = tmp_path / "first.png"+ first.write_bytes(b"PNG-FIRST")+ server = _FakeServer()+ _render(_FakeSlot(server), tmp_path, _params(init_img = str(first)))+ assert base64.b64decode(server.payloads[0]["init_image"]) == b"PNG-FIRST"+++def test_no_slot_or_kill_switch_or_reference_video_uses_the_cli(tmp_path, monkeypatch):+ assert (+ VideoBackend._h3_native_server_render(+ None,+ _Runtime(None),+ _params(),+ output_path = tmp_path / "o",+ on_log = print,+ cancel = threading.Event(),+ )+ is None+ )+ slot = _FakeSlot(_FakeServer())+ monkeypatch.setenv(h3.H3_NATIVE_SERVER_ENV, "0")+ assert _render(slot, tmp_path) is None+ assert slot.stopped == 1 # a server left from before the switch is released+ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV)+ server = _FakeServer()+ slot = _FakeSlot(server)+ assert _render(slot, tmp_path, _params(ref_videos = ("/frames",))) is None+ assert server.payloads == []+ # sd-cli reloads the whole model: a resident copy must not sit beside it.+ assert slot.released and slot.got == []+++def test_start_failure_falls_back_and_stays_one_shot(tmp_path, monkeypatch):+ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)+ slot = _FakeSlot(start_exc = RuntimeError("sd-server failed to become ready"))+ assert _render(slot, tmp_path) is None+ assert "start failed" in slot.disabled_reason+ # Disabled: the next render does not even try.+ slot.start_exc = AssertionError("must not start again")+ assert _render(slot, tmp_path) is None+++def test_unsupported_route_falls_back(tmp_path, monkeypatch):+ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)+ slot = _FakeSlot(_FakeServer(exc = SdCppServerUnsupported("no vid_gen")))+ assert _render(slot, tmp_path) is None+ assert slot.disabled_reason == "no vid_gen"+++def test_server_death_mid_render_falls_back(tmp_path, monkeypatch):+ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)+ slot = _FakeSlot(_FakeServer(exc = RuntimeError("connection lost"), alive_after = False))+ assert _render(slot, tmp_path) is None+ assert slot.disabled_reason.startswith("server died")+++def test_failed_job_on_a_live_server_raises_and_releases_it(tmp_path, monkeypatch):+ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)+ slot = _FakeSlot(_FakeServer(exc = RuntimeError("out of memory"), alive_after = True))+ with pytest.raises(RuntimeError, match = "out of memory"):+ _render(slot, tmp_path)+ assert slot.stopped == 1+ assert slot.disabled_reason is None # the next render respawns a server+++def test_cancel_is_a_cancellation_not_a_fallback(tmp_path, monkeypatch):+ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)+ cancel = threading.Event()+ cancel.set()+ slot = _FakeSlot(_FakeServer(exc = RuntimeError("server stopped"), alive_after = False))+ with pytest.raises(SdCppCancelled):+ _render(slot, tmp_path, cancel = cancel)+ assert slot.disabled_reason is None+++# -- slot lifecycle -----------------------------------------------------------------------------+++def test_sibling_server_binary(tmp_path):+ cli = tmp_path / "sd-cli"+ cli.write_text("")+ assert h3.h3_sibling_server_binary(str(cli)) is None+ (tmp_path / "sd-server").write_text("")+ assert h3.h3_sibling_server_binary(str(cli)) == str(tmp_path / "sd-server")+ assert h3.h3_sibling_server_binary(None) is None+++def test_slot_holds_the_managed_tree_while_its_server_lives(monkeypatch):+ started = []++ class _Srv:+ def __init__(self, binary):+ self.binary = binary+ self.alive = False+ self.stopped = False++ def start(+ self,+ files,+ *,+ offload = None,+ env = None,+ extra_args = None,+ ):+ started.append((files, offload, extra_args))+ self.alive = True++ def is_alive(self):+ return self.alive++ def stop(self):+ self.alive = False+ self.stopped = True++ monkeypatch.setattr(srv, "SdCppServer", _Srv)+ monkeypatch.setattr("core.inference.sd_cpp_engine.is_managed_binary", lambda b: True)+ monkeypatch.setattr(sd_cpp_backend, "_sd_cpp_backend", None, raising = False)+ slot = h3.H3NativeServerSlot(+ "/managed/sd-server", _FILES, ("--offload-to-cpu", "--diffusion-fa")+ )+ assert sd_cpp_backend._managed_tree_in_use() is False+ server = slot.get()+ assert slot.get() is server # reused, not respawned+ assert started == [(_FILES, ["--offload-to-cpu", "--diffusion-fa"], ["--rng", "cpu"])]+ assert sd_cpp_backend._managed_tree_in_use() is True+ with sd_cpp_backend._tree_claimed_for_install() as claimed:+ assert claimed is False # an install stands down for the resident server+ slot.stop()+ assert server.stopped is True+ assert sd_cpp_backend._managed_tree_in_use() is False+++def test_slot_respawns_a_dead_server(monkeypatch):+ class _Srv:+ def __init__(self, binary):+ self.alive = False++ def start(+ self,+ files,+ *,+ offload = None,+ env = None,+ extra_args = None,+ ):+ self.alive = True++ def is_alive(self):+ return self.alive++ def stop(self):+ self.alive = False++ monkeypatch.setattr(srv, "SdCppServer", _Srv)+ monkeypatch.setattr("core.inference.sd_cpp_engine.is_managed_binary", lambda b: False)+ slot = h3.H3NativeServerSlot("/x/sd-server", _FILES, ())+ first = slot.get()+ first.alive = False # crashed while idle+ second = slot.get()+ assert second is not first and second.is_alive()+ slot.stop()+++# -- spawn signature, idle timeout, pressure and release ------------------------------------------+++class _LifeSrv:+ """A fake SdCppServer that records every spawn."""++ spawns: list = []++ def __init__(self, binary):+ self.binary = binary+ self.alive = False++ def start(+ self,+ files,+ *,+ offload = None,+ env = None,+ extra_args = None,+ ):+ _LifeSrv.spawns.append({"offload": offload, "env": env, "extra_args": extra_args})+ self.alive = True++ def is_alive(self):+ return self.alive++ def stop(self):+ self.alive = False+++def life(monkeypatch):+ _LifeSrv.spawns = []+ monkeypatch.setattr(srv, "SdCppServer", _LifeSrv)+ monkeypatch.setattr("core.inference.sd_cpp_engine.is_managed_binary", lambda b: False)+ monkeypatch.delenv(h3.H3_NATIVE_SERVER_IDLE_ENV, raising = False)+ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)+ slots = []++ def make(**kwargs):+ slot = h3.H3NativeServerSlot("/x/sd-server", _FILES, ("--offload-to-cpu",), **kwargs)+ slots.append(slot)+ return slot++ yield make+ for slot in slots:+ slot.stop()+++RESIDENT = ["--diffusion-fa", "--backend", "diffusion=CUDA1,te=CUDA1,vae=CUDA1"]+OFFLOAD = ["--offload-to-cpu", "--diffusion-fa", "--backend", "diffusion=CUDA1,te=CUDA1,vae=CUDA1"]+SAGE_ENV = {h3.H3_QUANT_CUBLAS_ENV: h3.H3_QUANT_CUBLAS_MIN_BATCH}+++def test_slot_spawns_with_the_render_flags_and_env(life):+ slot = life()+ server = slot.get(RESIDENT + ["--sage-attn"], SAGE_ENV)+ assert _LifeSrv.spawns == [+ {"offload": RESIDENT + ["--sage-attn"], "env": SAGE_ENV, "extra_args": ["--rng", "cpu"]}+ ]+ # Same signature: reused, not respawned.+ assert slot.get(RESIDENT + ["--sage-attn"], dict(SAGE_ENV)) is server+ assert len(_LifeSrv.spawns) == 1+ assert slot.alive_signature()[2] == tuple(RESIDENT + ["--sage-attn"])+++ "first, second",+ [+ ((RESIDENT, {}), (OFFLOAD, {})), # memory decision changed+ ((RESIDENT, {}), (RESIDENT + ["--sage-attn"], SAGE_ENV)), # speed_mode=max switched on+ ((RESIDENT + ["--sage-attn"], SAGE_ENV), (RESIDENT + ["--sage-attn"], {})), # env differs+ ],+)+def test_slot_respawns_when_the_signature_changes(life, first, second):+ slot = life()+ old = slot.get(*first)+ new = slot.get(*second)+ assert new is not old+ assert old.is_alive() is False # never two resident copies+ assert [s["offload"] for s in _LifeSrv.spawns] == [first[0], second[0]]+ assert slot.last_release_reason == "signature changed"+++def test_slot_signature_covers_the_model_files(life):+ a = life()+ b = h3.H3NativeServerSlot(+ "/x/sd-server",+ SdCppModelFiles(+ diffusion_model = "/m/q8.gguf", vae = _FILES.vae, audio_vae = _FILES.audio_vae, llm = _FILES.llm+ ),+ )+ try:+ assert a._signature_for(tuple(RESIDENT), ()) != b._signature_for(tuple(RESIDENT), ())+ finally:+ b.stop()+++def test_idle_timeout_stops_the_server_and_leaves_no_timer(life, monkeypatch):+ monkeypatch.setenv(h3.H3_NATIVE_SERVER_IDLE_ENV, "0.05")+ slot = life()+ slot.begin_render()+ server = slot.get(RESIDENT, {})+ assert slot.end_render() is None+ timer = slot._timer+ assert timer is not None and timer.daemon+ timer.join(2.0)+ assert not timer.is_alive()+ assert server.is_alive() is False and slot.is_alive() is False+ assert slot.last_release_reason == "idle timeout"+ assert slot._timer is None+++def test_a_new_render_cancels_the_idle_timer(life, monkeypatch):+ monkeypatch.setenv(h3.H3_NATIVE_SERVER_IDLE_ENV, "0.2")+ slot = life()+ slot.begin_render()+ server = slot.get(RESIDENT, {})+ slot.end_render()+ first = slot._timer+ slot.begin_render() # the next render starts before the window ends+ assert slot._timer is None+ first.join(1.0)+ assert server.is_alive() is True+ # A stale firing (the timer raced the cancel) is ignored too.+ slot._on_idle(slot._timer_token - 1)+ assert server.is_alive() is True+ slot.end_render()+ slot.stop()+ assert slot._timer is None and server.is_alive() is False+++def test_stop_cancels_a_pending_timer(life, monkeypatch):+ monkeypatch.setenv(h3.H3_NATIVE_SERVER_IDLE_ENV, "30")+ slot = life()+ slot.begin_render()+ slot.get(RESIDENT, {})+ slot.end_render()+ timer = slot._timer+ slot.stop("unload")+ timer.join(1.0)+ assert not timer.is_alive()+++def test_idle_zero_stops_right_after_the_render(life, monkeypatch):+ monkeypatch.setenv(h3.H3_NATIVE_SERVER_IDLE_ENV, "0")+ slot = life()+ slot.begin_render()+ server = slot.get(RESIDENT, {})+ assert slot.end_render() == "idle timeout 0"+ assert server.is_alive() is False+++ "raw, expected", [("", 180.0), ("45", 45.0), ("-1", 180.0), ("abc", 180.0), ("0", 0.0)]+)+def test_idle_env_parsing(raw, expected):+ assert h3.h3_native_server_idle_s({h3.H3_NATIVE_SERVER_IDLE_ENV: raw}) == expected+++def test_memory_pressure_stops_the_server_after_the_render(life):+ reasons = iter(["free VRAM 1 MiB below the 4096 MiB reserve"])+ slot = life(pressure_probe = lambda: next(reasons))+ slot.begin_render()+ server = slot.get(RESIDENT, {})+ assert slot.end_render().startswith("memory pressure: free VRAM")+ assert server.is_alive() is False+ assert slot._timer is None+++def test_a_broken_probe_keeps_the_idle_bound(life, monkeypatch):+ monkeypatch.setenv(h3.H3_NATIVE_SERVER_IDLE_ENV, "30")++ def boom():+ raise OSError("nvidia-smi missing")++ slot = life(pressure_probe = boom)+ slot.begin_render()+ server = slot.get(RESIDENT, {})+ assert slot.end_render() is None+ assert server.is_alive() is True and slot._timer is not None+++def test_pressure_thresholds():+ gib = 1024**3+ ok = dict(+ vram_free = 40 * gib, vram_total = 80 * gib, host_available = 100 * gib, host_total = 200 * gib+ )+ assert h3.h3_native_server_pressure(**ok) is None+ # 15% of an 80 GiB card is 12 GiB.+ assert "VRAM" in h3.h3_native_server_pressure(**{**ok, "vram_free": 11 * gib})+ # A 16 GiB card: the 4 GiB floor wins over 15%.+ assert "VRAM" in h3.h3_native_server_pressure(+ **{**ok, "vram_free": 3 * gib, "vram_total": 16 * gib}+ )+ assert "host RAM" in h3.h3_native_server_pressure(**{**ok, "host_available": 29 * gib})+ # Unknown readings decide nothing.+ assert (+ h3.h3_native_server_pressure(+ vram_free = None, vram_total = None, host_available = None, host_total = None+ )+ is None+ )+++def test_release_stops_an_idle_server_and_defers_a_busy_one(life, monkeypatch):+ monkeypatch.setenv(h3.H3_NATIVE_SERVER_IDLE_ENV, "30")+ idle = life()+ idle.begin_render()+ idle_server = idle.get(RESIDENT, {})+ idle.end_render()+ busy = life()+ busy.begin_render()+ busy_server = busy.get(RESIDENT, {})+ assert h3.release_h3_native_servers("export subprocess starting") >= 2+ assert idle_server.is_alive() is False and idle._timer is None+ assert idle.last_release_reason == "export subprocess starting"+ # The render in flight finishes on its server; the server goes as it ends.+ assert busy_server.is_alive() is True+ assert busy.end_render() == "export subprocess starting"+ assert busy_server.is_alive() is False+++def test_gpu_arbiter_releases_idle_video_servers_for_other_owners(monkeypatch):+ from core.inference import gpu_arbiter++ seen = []+ monkeypatch.setattr(h3, "release_h3_native_servers", lambda reason: seen.append(reason) or 0)+ monkeypatch.setattr(gpu_arbiter, "_owner", None)+ monkeypatch.setattr(gpu_arbiter, "raise_if_other_accounts_active", lambda *a, **k: None)+ gpu_arbiter.acquire_for(gpu_arbiter.CHAT, account_id = "a")+ assert seen == ["GPU acquired for chat"]+ gpu_arbiter.acquire_for(gpu_arbiter.VIDEO, account_id = "a", allow_evict = True)+ assert seen == ["GPU acquired for chat"] # the video owner itself keeps its server+++def test_render_hands_the_cli_flags_and_env_to_the_slot(tmp_path, monkeypatch):+ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)+ slot = _FakeSlot(_FakeServer())+ _render(slot, tmp_path, flags = RESIDENT + ["--sage-attn"], env = SAGE_ENV)+ assert slot.got == [(RESIDENT + ["--sage-attn"], SAGE_ENV)]+ assert slot.renders == 1 and slot.ended == 1+++def test_render_ends_the_slot_render_on_failure_too(tmp_path, monkeypatch):+ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)+ slot = _FakeSlot(_FakeServer(exc = RuntimeError("out of memory"), alive_after = True))+ with pytest.raises(RuntimeError):+ _render(slot, tmp_path)+ assert slot.ended == 1+++def test_kill_switch_stops_a_live_server_and_uses_the_cli(tmp_path, monkeypatch, life):+ slot = life()+ slot.begin_render()+ server = slot.get(RESIDENT, {})+ slot.end_render()+ monkeypatch.setenv(h3.H3_NATIVE_SERVER_ENV, "0")+ assert _render(slot, tmp_path) is None+ assert server.is_alive() is False and slot._timer is None+++# -- through VideoBackend.generate ----------------------------------------------------------------+++def _generate_backend(+ monkeypatch,+ tmp_path,+ slot,+ *,+ env = (),+):+ import dataclasses++ from test_h3_native_resident import AUTO_FLAGS, _backend_with_files++ calls: list = []+ backend = _backend_with_files(+ monkeypatch, tmp_path, calls, memory_mode = "auto", flags = AUTO_FLAGS+ )+ backend._state = dataclasses.replace(+ backend._state,+ pipe = dataclasses.replace(backend._state.pipe, server_slot = slot, env = tuple(env)),+ )+ return backend, calls, AUTO_FLAGS+++class _StartFailsSlot(_FakeSlot):+ def __init__(self, live = None):+ super().__init__(start_exc = RuntimeError("no server"))+ self.live = live++ def alive_signature(self):+ return self.live+++def test_generate_gives_the_server_and_the_fallback_cli_the_same_flags_and_env(+ monkeypatch, tmp_path+):+ import core.inference.video as video++ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)+ gib = 1024**3+ monkeypatch.setattr(video, "_h3_card_free_bytes", lambda device, ordinal: 90 * gib)+ slot = _StartFailsSlot()+ backend, calls, auto = _generate_backend(+ monkeypatch, tmp_path, slot, env = tuple(SAGE_ENV.items())+ )+ backend.generate(prompt = "a fox", width = 960, height = 544)+ resident = [f for f in auto if f not in ("--offload-to-cpu", "--stream-layers")]+ assert slot.got == [(resident, SAGE_ENV)]+ assert calls[0]["offload"] == resident and calls[0]["env"] == SAGE_ENV+++def test_a_live_resident_server_is_not_pushed_back_to_offload_by_its_own_usage(+ monkeypatch, tmp_path+):+ import core.inference.video as video++ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)+ from test_h3_native_resident import AUTO_FLAGS++ gib = 1024**3+ resident = tuple(f for f in AUTO_FLAGS if f not in ("--offload-to-cpu", "--stream-layers"))+ slot = _StartFailsSlot(live = ("/x/sd-server", (), resident, ()))+ # The live server holds the 32 GiB bundle, so the card reads only 10 GiB free: enough for this clip's activations.+ monkeypatch.setattr(video, "_h3_card_free_bytes", lambda *_a: 10 * gib)+ backend, calls, _ = _generate_backend(monkeypatch, tmp_path, slot)+ result = backend.generate(prompt = "a fox", width = 960, height = 544)+ assert slot.got[0][0] == list(resident)+ assert result["offload_policy"] == "none"+++def test_a_live_resident_server_does_not_hide_a_clip_too_large_for_the_card(monkeypatch, tmp_path):+ import core.inference.video as video++ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)+ from test_h3_native_resident import AUTO_FLAGS++ gib = 1024**3+ resident = tuple(f for f in AUTO_FLAGS if f not in ("--offload-to-cpu", "--stream-layers"))+ slot = _StartFailsSlot(live = ("/x/sd-server", (), resident, ()))+ monkeypatch.setattr(video, "_h3_card_free_bytes", lambda *_a: 10 * gib)+ backend, calls, _ = _generate_backend(monkeypatch, tmp_path, slot)+ # 8x the pixel volume: ~43 GiB of activations on top of the weights, more than the 10 GiB the card has left.+ result = backend.generate(prompt = "a fox", width = 1920, height = 1088, num_frames = 241)+ assert "--offload-to-cpu" in slot.got[0][0]+ assert result["offload_policy"] == "group"+++def test_release_does_not_wait_on_a_server_start(life, monkeypatch):+ monkeypatch.setenv(h3.H3_NATIVE_SERVER_IDLE_ENV, "30")+ slot = life()+ slot.begin_render()+ server = slot.get(RESIDENT, {})+ holder_ready, done = threading.Event(), threading.Event()++ def hold(): # stands in for a render spawning its server under the slot lock+ with slot._lock:+ holder_ready.set()+ done.wait(5)++ th = threading.Thread(target = hold)+ th.start()+ holder_ready.wait(2)+ import time as _time++ t0 = _time.monotonic()+ assert slot.release("chat load") is True+ assert _time.monotonic() - t0 < 2.0+ done.set()+ th.join(2)+ # The pending release is honoured when that render ends.+ assert slot.end_render() == "chat load"+ assert server.is_alive() is False+++def test_cancel_aborts_a_server_start_in_progress(monkeypatch):+ """Unload and Cancel wait on the render, which holds the generate lock through the server's model load."""+ import time as _time++ monkeypatch.setattr("core.inference.sd_cpp_engine.is_managed_binary", lambda b: False)+ loading = threading.Event()++ class _SlowSrv:+ def __init__(self, binary):+ self.aborted = threading.Event()+ self.stops = 0++ def start(+ self,+ files,+ *,+ offload = None,+ env = None,+ extra_args = None,+ ):+ loading.set()+ if not self.aborted.wait(10):+ raise AssertionError("start was never aborted")+ raise SdCppCancelled("sd-server startup was cancelled.")++ def stop(self):+ self.stops += 1+ self.aborted.set()++ def is_alive(self):+ return False++ monkeypatch.setattr(srv, "SdCppServer", _SlowSrv)+ slot = h3.H3NativeServerSlot("/x/sd-server", _FILES)+ cancel = threading.Event()+ threading.Timer(0.3, cancel.set).start()+ t0 = _time.monotonic()+ with pytest.raises(SdCppCancelled):+ slot.get(RESIDENT, {}, cancel_event = cancel)+ assert loading.is_set() and _time.monotonic() - t0 < 5+ assert slot.alive_signature() is None and slot.disabled_reason is None+ # Already cancelled: nothing is spawned at all.+ with pytest.raises(SdCppCancelled):+ slot.get(RESIDENT, {}, cancel_event = cancel)+++def test_a_live_resident_server_keeps_the_committed_offload_when_the_card_is_unreadable(+ monkeypatch, tmp_path+):+ import core.inference.video as video++ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)+ from test_h3_native_resident import AUTO_FLAGS++ resident = tuple(f for f in AUTO_FLAGS if f not in ("--offload-to-cpu", "--stream-layers"))+ slot = _StartFailsSlot(live = ("/x/sd-server", (), resident, ()))+ monkeypatch.setattr(video, "_h3_card_free_bytes", lambda *_a: None)+ backend, calls, _ = _generate_backend(monkeypatch, tmp_path, slot)+ backend.generate(prompt = "a fox", width = 960, height = 544)+ assert slot.got[0][0] == list(AUTO_FLAGS)+++def test_host_reserve_is_a_share_of_the_cgroup_limit(monkeypatch):+ import core.inference.diffusion_memory as dm+ import core.inference.video as video++ gib_mib = 1024+ monkeypatch.setattr(video, "_h3_card_memory_bytes", lambda *_a: (None, None))+ # A 32 GiB container on a 512 GiB host with 20 GiB still chargeable: 15% of 32 GiB is under 20 GiB.+ monkeypatch.setattr(dm, "_system_memory_mib", lambda: (512 * gib_mib, 400 * gib_mib))+ monkeypatch.setattr(dm, "_available_system_memory_mib", lambda: 20 * gib_mib)+ monkeypatch.setattr(dm, "_cgroup_memory_limit_mib", lambda: 32 * gib_mib)+ assert video._h3_native_server_pressure("cuda", None) is None+ # Without a limit the physical total stands.+ monkeypatch.setattr(dm, "_cgroup_memory_limit_mib", lambda: None)+ assert "host RAM" in video._h3_native_server_pressure("cuda", None)+++def test_the_tree_stays_held_until_the_server_has_exited(monkeypatch):+ monkeypatch.setattr("core.inference.sd_cpp_engine.is_managed_binary", lambda b: True)+ seen = []++ class _ExitingSrv:+ def __init__(self, binary):+ self.alive = False++ def start(+ self,+ files,+ *,+ offload = None,+ env = None,+ extra_args = None,+ ):+ self.alive = True++ def is_alive(self):+ return self.alive++ def stop(self):+ # SIGTERM sent, process not reaped yet: an install must still stand down.+ seen.append(sd_cpp_backend._external_tree_holder_alive())+ self.alive = False++ monkeypatch.setattr(srv, "SdCppServer", _ExitingSrv)+ slot = h3.H3NativeServerSlot("/x/sd-server", _FILES)+ slot.get(RESIDENT, {})+ slot.release("chat load")+ assert seen == [True]+ assert sd_cpp_backend._external_tree_holder_alive() is False