Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep MiniMax-H3 GGUF renders on a resident sd-server, released when idle (#12461)

Daniel Hanc29edef883dd
diff --git a/studio/backend/core/export/orchestrator.py b/studio/backend/core/export/orchestrator.py
index f7d5dc15ef2..c712f47f8d8 100644
--- a/studio/backend/core/export/orchestrator.py
+++ b/studio/backend/core/export/orchestrator.py
@@ -207,6 +207,12 @@ class ExportOrchestrator:
return True
def _spawn_subprocess(self, config: dict) -> None:
+ # Export does not evict loaded models; at least free an idle resident H3 sd-server.
+ try:
+ from core.inference.video_minimax_h3 import release_h3_native_servers
+ release_h3_native_servers("export subprocess starting")
+ except Exception as exc: # noqa: BLE001 - never block an export on this
+ logger.warning("Could not release the idle video sd-server for export: %s", exc)
# Inside an op a reservation is an install about to abort on is_export_active(), so raising here
# would kill the export for an install that never proceeds.
from utils.transformers_version import sidecar_swap_in_progress
diff --git a/studio/backend/core/inference/gpu_arbiter.py b/studio/backend/core/inference/gpu_arbiter.py
index cd5392c5054..f829ed6a206 100644
--- a/studio/backend/core/inference/gpu_arbiter.py
+++ b/studio/backend/core/inference/gpu_arbiter.py
@@ -72,6 +72,14 @@ def _evict_diffusion() -> None:
get_active_diffusion_engine().unload()
+def _release_idle_video_servers(reason: str) -> None:
+ try:
+ from core.inference.video_minimax_h3 import release_h3_native_servers
+ release_h3_native_servers(reason)
+ except Exception as exc: # noqa: BLE001 - never block an acquire on this
+ logger.warning("gpu_arbiter: could not release the idle video sd-server: %s", exc)
+
+
def _evict_video() -> None:
from core.inference.video import get_video_backend
get_video_backend().unload()
@@ -192,6 +200,9 @@ def acquire_for(
raise GpuBusyForAnotherAccountError(_owner, busy)
logger.info("gpu_arbiter: evicting %s for %s", _owner, owner)
_EVICTORS[_owner]()
+ if owner != VIDEO:
+ # A resident H3 sd-server sits outside every owner's teardown.
+ _release_idle_video_servers(f"GPU acquired for {owner}")
# Records who LOADED the model; a plain re-assert must not hand it to whoever asked last.
claims = _owner != owner or register is not None or replacing
_owner = owner
diff --git a/studio/backend/core/inference/sd_cpp_args.py b/studio/backend/core/inference/sd_cpp_args.py
index fef1045cb5e..ce8979c81ea 100644
--- a/studio/backend/core/inference/sd_cpp_args.py
+++ b/studio/backend/core/inference/sd_cpp_args.py
@@ -552,6 +552,8 @@ def build_sd_cpp_server_command(
cmd: list[str] = [binary, "--diffusion-model", files.diffusion_model]
for flag, value in (
("--vae", files.vae),
+ # Without it a resident H3 vid_gen server renders a silent clip.
+ ("--audio-vae", files.audio_vae),
("--clip_l", files.clip_l),
("--clip_g", files.clip_g),
("--t5xxl", files.t5xxl),
@@ -653,6 +655,59 @@ def build_img_gen_request(
return req
+def h3_server_eligible(params: "SdCppVideoGenParams") -> bool:
+ """False for reference videos / audio: path-only sd-cli flags with no ``vid_gen`` JSON field."""
+ return not (params.ref_videos or params.ref_video_audios or params.ref_audios)
+
+
+def build_vid_gen_request(
+ params: "SdCppVideoGenParams",
+ *,
+ images_b64: Optional[dict[str, str]] = None,
+ ref_images_b64: Optional[list[str]] = None,
+ output_compression: int = 90,
+) -> dict:
+ """``POST /sdcpp/v1/vid_gen`` body equivalent to ``build_sd_cpp_video_command`` (same pixels for the same seed).
+
+ AVI: needs no WebM build, as the CUDA prebuilt's sd-cli writes. ``--rng cpu`` is a server context flag, passed at
+ spawn."""
+ if not (params.prompt or "").strip():
+ raise ValueError("prompt is required")
+ if params.width <= 0 or params.height <= 0 or params.num_frames <= 0:
+ raise ValueError("width, height, and num_frames must be positive")
+ if not h3_server_eligible(params):
+ raise ValueError("reference videos and audio are not accepted by the sd-server vid_gen API")
+ images = dict(images_b64 or {})
+ if (images.get("init_image") or images.get("end_image")) and ref_images_b64:
+ raise ValueError(
+ "MiniMax-H3 keyframes and references cannot be combined: they run against "
+ "different denoiser partitions."
+ )
+ sample_params: dict = {"guidance": {"txt_cfg": float(params.cfg_scale)}}
+ if params.steps is not None:
+ sample_params["sample_steps"] = int(params.steps)
+ if params.flow_shift is not None:
+ sample_params["flow_shift"] = float(params.flow_shift)
+ req: dict = {
+ "prompt": params.prompt,
+ "width": int(params.width),
+ "height": int(params.height),
+ "video_frames": int(params.num_frames),
+ "fps": int(params.fps),
+ "sample_params": sample_params,
+ "output_format": "avi",
+ "output_compression": int(output_compression),
+ }
+ if params.seed is not None:
+ req["seed"] = int(params.seed)
+ for key in ("init_image", "end_image"):
+ if images.get(key):
+ req[key] = images[key]
+ if ref_images_b64:
+ req["ref_images"] = list(ref_images_b64)
+ return req
+
+
def _fmt_float(value: float) -> str:
"""Compact float -> str: drop a trailing ``.0`` (``1.0`` -> ``1``) to keep logged commands tidy."""
f = float(value)
diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.py
index df958248c36..e94b068d6ad 100644
--- a/studio/backend/core/inference/sd_cpp_backend.py
+++ b/studio/backend/core/inference/sd_cpp_backend.py
@@ -26,6 +26,7 @@ import re
import sys
import threading
import time
+import weakref
from dataclasses import dataclass, field, replace
from pathlib import Path
from typing import Any, Optional
@@ -1568,7 +1569,33 @@ def _managed_tree_in_use() -> bool:
Reads the singleton without a lock on purpose: a stale answer either defers an upgrade to the
next load (harmless) or lets one through in a window the load path guards anyway.
"""
- return _tree_in_use(_sd_cpp_backend)
+ return _tree_in_use(_sd_cpp_backend) or _external_tree_holder_alive()
+
+
+# Other backends' processes running out of the managed tree (the H3 video sd-server), so an install stands down for
+# them too. Weak: a dropped runtime cannot pin the tree.
+_external_tree_holders: "weakref.WeakSet[Any]" = weakref.WeakSet()
+
+
+def register_tree_holder(holder: Any) -> None:
+ with _tree_state:
+ _external_tree_holders.add(holder)
+
+
+def unregister_tree_holder(holder: Any) -> None:
+ with _tree_state:
+ _external_tree_holders.discard(holder)
+ _tree_state.notify_all()
+
+
+def _external_tree_holder_alive() -> bool:
+ for holder in list(_external_tree_holders):
+ try:
+ if holder.is_alive():
+ return True
+ except Exception: # noqa: BLE001 -- a broken holder must not wedge installs either way
+ continue
+ return False
def _accelerator_changed(binary: str, accelerator: str) -> bool:
diff --git a/studio/backend/core/inference/sd_cpp_server.py b/studio/backend/core/inference/sd_cpp_server.py
index 5076c29abba..8d1d23b58f0 100644
--- a/studio/backend/core/inference/sd_cpp_server.py
+++ b/studio/backend/core/inference/sd_cpp_server.py
@@ -15,6 +15,8 @@ Shape mirrors ``core/rag/embed_llama_server.py``:
drain stdout on a daemon thread, poll until ready.
* ``img_gen`` -- POST ``/sdcpp/v1/img_gen`` (the whole batch in one request),
poll the async job to a terminal state, return image bytes.
+ * ``vid_gen`` -- POST ``/sdcpp/v1/vid_gen`` (MiniMax-H3 native video), same polling,
+ return the encoded container (video + muxed audio).
* ``stop`` -- SIGTERM -> wait -> SIGKILL, join the drain thread (idempotent).
Readiness is real: upstream ``main.cpp`` loads the model BEFORE it binds the port and
@@ -78,6 +80,7 @@ _TRANSPORT_ERRORS = (
# capabilities endpoint (which can block).
_READY_PATH = "/v1/models"
_IMG_GEN_PATH = "/sdcpp/v1/img_gen"
+_VID_GEN_PATH = "/sdcpp/v1/vid_gen"
_JOBS_PATH = "/sdcpp/v1/jobs"
# Stable parameter residency reported once during startup. Compute-buffer lines are deliberately excluded: those
@@ -155,6 +158,17 @@ def _has_ancestor(
return False
+class SdCppServerUnsupported(RuntimeError):
+ """The sd-server has no route / mode for this job kind; nothing ran, so the caller can fall back to sd-cli."""
+
+
+def _route_unsupported(status_code: int, text: str, kind: str) -> bool:
+ # 404: no route in this build; 400 "does not support": the loaded model lacks the mode (runtime.cpp).
+ if status_code == 404:
+ return True
+ return status_code == 400 and f"does not support {kind}" in (text or "")
+
+
class SdCppServer:
"""A resident ``sd-server`` subprocess plus the HTTP client that drives it."""
@@ -525,6 +539,62 @@ class SdCppServer:
) -> list[bytes]:
"""Submit one async ``img_gen`` job, poll it to completion, return image bytes.
+ ``on_step`` receives each server stdout line (for the step bar). ``cancel_event``,
+ when set, cancels the job via the native endpoint and raises ``SdCppCancelled``.
+ Raises ``RuntimeError`` on submit/poll failures (including the server dying), with
+ the log tail attached.
+ """
+ return self._run_job(
+ "img_gen",
+ _IMG_GEN_PATH,
+ payload,
+ self._decode_images,
+ on_step = on_step,
+ cancel_event = cancel_event,
+ poll_interval = poll_interval,
+ submit_timeout = submit_timeout,
+ total_timeout = total_timeout,
+ )
+
+ def vid_gen(
+ self,
+ payload: dict[str, Any],
+ *,
+ on_step: Optional[Callable[[str], None]] = None,
+ cancel_event: Optional[threading.Event] = None,
+ poll_interval: float = 0.4,
+ submit_timeout: float = 60.0,
+ total_timeout: float = NATIVE_GENERATION_TIMEOUT_S,
+ ) -> bytes:
+ """``img_gen``'s contract for one ``vid_gen`` job, returning the container bytes (audio muxed); raises
+ ``SdCppServerUnsupported`` when the server cannot do video."""
+ return self._run_job(
+ "vid_gen",
+ _VID_GEN_PATH,
+ payload,
+ self._decode_video,
+ on_step = on_step,
+ cancel_event = cancel_event,
+ poll_interval = poll_interval,
+ submit_timeout = submit_timeout,
+ total_timeout = total_timeout,
+ )
+
+ def _run_job(
+ self,
+ kind: str,
+ path: str,
+ payload: dict[str, Any],
+ decode: Callable[[dict[str, Any]], Any],
+ *,
+ on_step: Optional[Callable[[str], None]] = None,
+ cancel_event: Optional[threading.Event] = None,
+ poll_interval: float = 0.4,
+ submit_timeout: float = 60.0,
+ total_timeout: float = NATIVE_GENERATION_TIMEOUT_S,
+ ) -> Any:
+ """Submit one async ``kind`` job at ``path``, poll it to completion, return ``decode(job)``.
+
``on_step`` receives each server stdout line (for the step bar). ``cancel_event``,
when set, cancels the job via the native endpoint and raises ``SdCppCancelled``.
Raises ``RuntimeError`` on submit/poll failures (including the server dying), with
@@ -541,29 +611,34 @@ class SdCppServer:
try:
try:
resp = self._client.post(
- f"{self.base_url}{_IMG_GEN_PATH}", json = payload, timeout = submit_timeout
+ f"{self.base_url}{path}", json = payload, timeout = submit_timeout
)
except (*_TRANSPORT_ERRORS, httpx.TimeoutException) as exc:
- raise RuntimeError(self._died_message("img_gen submit", exc)) from exc
+ raise RuntimeError(self._died_message(f"{kind} submit", exc)) from exc
if resp.status_code == 429:
raise RuntimeError("sd-server job queue is full (HTTP 429).")
if resp.status_code not in (200, 202):
+ if _route_unsupported(resp.status_code, resp.text, kind):
+ raise SdCppServerUnsupported(
+ f"this sd-server does not serve {kind} (HTTP {resp.status_code}): "
+ f"{resp.text[:300]}"
+ )
raise RuntimeError(
- f"sd-server img_gen submit -> {resp.status_code}: {resp.text[:500]}"
+ f"sd-server {kind} submit -> {resp.status_code}: {resp.text[:500]}"
)
try:
job = resp.json()
except ValueError as exc:
raise RuntimeError(
- f"sd-server img_gen returned a non-JSON submit response: {exc}"
+ f"sd-server {kind} returned a non-JSON submit response: {exc}"
) from exc
if not isinstance(job, dict):
raise RuntimeError(
- f"sd-server img_gen returned an unexpected submit response type: {type(job)}"
+ f"sd-server {kind} returned an unexpected submit response type: {type(job)}"
)
job_id = job.get("id")
if not job_id:
- raise RuntimeError(f"sd-server img_gen returned no job id: {job}")
+ raise RuntimeError(f"sd-server {kind} returned no job id: {job}")
deadline = time.monotonic() + total_timeout
cancel_sent_at: Optional[float] = None
@@ -583,7 +658,7 @@ class SdCppServer:
# Unwinding a cancel (e.g. unload killed the server): clean cancellation.
if cancel_event is not None and cancel_event.is_set():
raise SdCppCancelled("sd-server generation was cancelled.")
- raise RuntimeError(self._died_message("img_gen poll", None))
+ raise RuntimeError(self._died_message(f"{kind} poll", None))
if time.monotonic() > deadline:
# sd-server will not interrupt an in-flight job, so cancel + stop to free the slot; the backend
# reloads on the next generate.
@@ -616,13 +691,19 @@ class SdCppServer:
)
status = jd.get("status")
if status == _TERMINAL_OK:
- return self._decode_images(jd)
+ return decode(jd)
if status == _TERMINAL_FAIL:
err = jd.get("error") or {}
- raise RuntimeError(
+ message = (
"sd-server generation failed: "
f"{err.get('code', 'error')}: {err.get('message', '')}".strip()
)
+ if kind == "vid_gen":
+ # The job error is generic; the cause (OOM, abort) is only in the log.
+ tail = _diagnostic_tail(self._tail)
+ if tail:
+ message += "\nLast output:\n" + tail[:1500]
+ raise RuntimeError(message)
if status == _TERMINAL_CANCELLED:
raise SdCppCancelled("sd-server generation was cancelled.")
time.sleep(poll_interval)
@@ -636,6 +717,17 @@ class SdCppServer:
except Exception: # noqa: BLE001 -- cancel is best-effort
pass
+ @staticmethod
+ def _decode_video(job: dict[str, Any]) -> bytes:
+ result = job.get("result") if isinstance(job, dict) else None
+ b64 = result.get("b64_json") if isinstance(result, dict) else None
+ if not b64:
+ raise RuntimeError("sd-server completed the job but returned no video.")
+ try:
+ return base64.b64decode(b64)
+ except Exception as exc: # noqa: BLE001
+ raise RuntimeError(f"sd-server returned an undecodable video: {exc}") from exc
+
@staticmethod
def _decode_images(job: dict[str, Any]) -> list[bytes]:
# Type-check each level before indexing (guards a misbehaving/older server).
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index d9debeeac88..44b1ec385a2 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -42,7 +42,7 @@ import time
import types
from dataclasses import dataclass
from pathlib import Path
-from typing import Any, Optional
+from typing import Any, Callable, Optional
from hub.utils.hf_errors import modelscope_missing
from loggers import get_logger
@@ -1290,10 +1290,13 @@ def _h3_free_device_bytes(device: str) -> Optional[int]:
return None
-def _h3_card_free_bytes(device: Optional[str], ordinal: Optional[int]) -> Optional[int]:
- """Live free VRAM via nvidia-smi / amd-smi (no CUDA context here), or None. No pin = sd.cpp's ordinal 0."""
+def _h3_card_memory_bytes(
+ device: Optional[str], ordinal: Optional[int]
+) -> tuple[Optional[int], Optional[int]]:
+ """(free, total) VRAM of the H3 render card via nvidia-smi / amd-smi (no CUDA context), or (None, None). No pin =
+ ordinal 0."""
if device != "cuda":
- return None
+ return None, None
try:
from utils.hardware import get_visible_gpu_utilization, gpu_query
@@ -1306,11 +1309,44 @@ def _h3_card_free_bytes(device: Optional[str], ordinal: Optional[int]) -> Option
total = entry.get("vram_total_gb")
used = entry.get("vram_used_gb")
if total is None or used is None:
- return None
- return max(0, int((float(total) - float(used)) * 1024**3))
+ return None, None
+ total_bytes = int(float(total) * 1024**3)
+ return max(0, int((float(total) - float(used)) * 1024**3)), total_bytes
except Exception: # noqa: BLE001 -- an unreadable card decides nothing
- return None
- return None
+ return None, None
+ return None, None
+
+
+def _h3_card_free_bytes(device: Optional[str], ordinal: Optional[int]) -> Optional[int]:
+ """Free VRAM of the H3 render card, or None (keeps the committed offload flags)."""
+ return _h3_card_memory_bytes(device, ordinal)[0]
+
+
+def _h3_native_server_pressure(device: Optional[str], ordinal: Optional[int]) -> Optional[str]:
+ """Why the resident H3 sd-server should not outlive a render (``h3_native_server_pressure``), or None."""
+ from .diffusion_memory import (
+ _available_system_memory_mib,
+ _cgroup_memory_limit_mib,
+ _system_memory_mib,
+ )
+ from .video_minimax_h3 import h3_native_server_pressure
+
+ vram_free, vram_total = _h3_card_memory_bytes(device, ordinal)
+ try:
+ host_total_mib = _system_memory_mib()[0]
+ # Available RAM is cgroup-capped, so the reserve must be a share of the same limit.
+ cgroup_limit_mib = _cgroup_memory_limit_mib()
+ if cgroup_limit_mib and (host_total_mib is None or cgroup_limit_mib < host_total_mib):
+ host_total_mib = cgroup_limit_mib
+ host_avail_mib = _available_system_memory_mib()
+ except Exception: # noqa: BLE001 -- unknown host memory decides nothing
+ host_total_mib = host_avail_mib = None
+ return h3_native_server_pressure(
+ vram_free = vram_free,
+ vram_total = vram_total,
+ host_available = None if host_avail_mib is None else int(host_avail_mib) << 20,
+ host_total = None if host_total_mib is None else int(host_total_mib) << 20,
+ )
def _h3_device_capacity_bytes(device: str) -> Optional[int]:
@@ -3533,22 +3569,41 @@ class VideoBackend:
# After the policy, so the pin can see which modules it left on the CPU; without it sd.cpp uses ordinal 0
# whatever was selected.
native_offload += tuple(device_backend_flags(native_device_name, list(native_offload)))
- from .video_minimax_h3 import MiniMaxH3NativeRuntime
+ from .video_minimax_h3 import (
+ H3NativeServerSlot,
+ MiniMaxH3NativeRuntime,
+ h3_sibling_server_binary,
+ )
+ native_files = SdCppModelFiles(
+ diffusion_model = str(resolved[0]),
+ llm = str(resolved[1]),
+ vae = str(resolved[2]),
+ audio_vae = str(resolved[3]),
+ )
+ # Nothing spawns here: the slot starts the server on the first render; no sibling sd-server keeps one-shot.
+ server_binary = h3_sibling_server_binary(getattr(engine, "binary", None))
runtime = MiniMaxH3NativeRuntime(
engine = engine,
# Pinned under the reader claim above, where this exact file answered --help with the H3 options. Taking it
# at generation time instead would compare a replacement against itself.
binary_identity = binary_identity,
selected_card = selected_card,
- files = SdCppModelFiles(
- diffusion_model = str(resolved[0]),
- llm = str(resolved[1]),
- vae = str(resolved[2]),
- audio_vae = str(resolved[3]),
- ),
+ files = native_files,
offload_flags = native_offload,
env = native_env,
+ server_slot = (
+ H3NativeServerSlot(
+ server_binary,
+ native_files,
+ native_offload,
+ pressure_probe = functools.partial(
+ _h3_native_server_pressure, native_device, native_ordinal
+ ),
+ )
+ if server_binary
+ else None
+ ),
)
with self._lock:
@@ -8835,6 +8890,97 @@ class VideoBackend:
audios = decoded_audios,
)
+ def _h3_native_server_render(
+ self,
+ runtime: Any,
+ params: Any,
+ *,
+ output_path: Path,
+ on_log: Callable[[str], None],
+ cancel: threading.Event,
+ flags: Optional[list[str]] = None,
+ env: Optional[dict[str, str]] = None,
+ ) -> Optional[Path]:
+ """Render one MiniMax-H3 clip on the runtime's resident sd-server, spawned with this render's sd-cli ``flags`` /
+ ``env``.
+
+ Returns the written container, or None for the one-shot sd-cli: no slot, kill switch, reference video / audio,
+ or a server that cannot start, lacks vid_gen, or died (the runtime then stays one-shot). Every fallback first
+ stops the server so the two never hold the model at once. A job the live server fails is raised after stopping
+ it. Called inside the generation's managed-tree reader claim."""
+ import base64
+
+ from .sd_cpp_args import build_vid_gen_request, h3_server_eligible
+ from .sd_cpp_engine import SdCppCancelled
+ from .sd_cpp_server import SdCppServerUnsupported
+ from .video_minimax_h3 import h3_native_server_enabled
+
+ slot = getattr(runtime, "server_slot", None)
+ if slot is None or slot.disabled_reason is not None:
+ return None
+ if not h3_native_server_enabled():
+ slot.stop("UNSLOTH_H3_NATIVE_SERVER=0")
+ return None
+ if not h3_server_eligible(params):
+ # The sd-cli loads the whole model again; a resident copy beside it may not fit.
+ slot.release("reference video / audio render runs on sd-cli")
+ return None
+
+ def b64_file(path: Optional[str]) -> Optional[str]:
+ if not path:
+ return None
+ return base64.b64encode(Path(path).read_bytes()).decode("ascii")
+
+ payload = build_vid_gen_request(
+ params,
+ images_b64 = {
+ "init_image": b64_file(params.init_img),
+ "end_image": b64_file(params.end_img),
+ },
+ ref_images_b64 = [b64_file(p) for p in params.ref_images] or None,
+ )
+ render_flags = list(runtime.offload_flags) if flags is None else list(flags)
+ render_env = dict(runtime.env) if env is None else dict(env)
+ slot.begin_render()
+ try:
+ try:
+ server = slot.get(render_flags, render_env, cancel_event = cancel)
+ except SdCppCancelled:
+ raise
+ except Exception as exc: # noqa: BLE001 - any start failure leaves the one-shot path
+ if cancel.is_set():
+ raise SdCppCancelled("sd-server start was cancelled.") from exc
+ logger.warning("video.h3_native_server_start_failed (using sd-cli): %s", exc)
+ slot.disable(f"start failed: {exc}")
+ return None
+ try:
+ data = server.vid_gen(payload, on_step = on_log, cancel_event = cancel)
+ except SdCppServerUnsupported as exc:
+ logger.warning("video.h3_native_server_unsupported (using sd-cli): %s", exc)
+ slot.disable(str(exc))
+ return None
+ except SdCppCancelled:
+ raise
+ except RuntimeError as exc:
+ if cancel.is_set():
+ raise SdCppCancelled("sd-server generation was cancelled.") from exc
+ if not server.is_alive():
+ logger.warning(
+ "video.h3_native_server_died (retrying once on sd-cli, staying one-shot): %s",
+ exc,
+ )
+ slot.disable(f"server died: {exc}")
+ return None
+ slot.stop("render failed")
+ raise
+ Path(output_path).write_bytes(data)
+ return Path(output_path)
+ finally:
+ try:
+ slot.end_render()
+ except Exception as exc: # noqa: BLE001 - the render's own outcome wins
+ logger.warning("video.h3_native_server_release_failed: %s", exc)
+
def _generate_h3_native(
self,
*,
@@ -8882,10 +9028,30 @@ class VideoBackend:
need = h3_native_resident_bytes(file_bytes, width, height, frames)
except Exception: # noqa: BLE001 -- unknown size keeps the committed offload
need = None
+ free = None
+ if need:
+ # A live resident server's weights count as free; a larger clip's activations must still fit the rest.
+ slot = getattr(runtime, "server_slot", None)
+ live = slot.alive_signature() if slot is not None else None
+ resident_flags, _ = h3_native_render_flags(
+ runtime.offload_flags,
+ memory_mode = state.memory_mode,
+ free_bytes = need,
+ need_bytes = need,
+ )
+ if (
+ live is not None
+ and live[2] == tuple(resident_flags)
+ and "--offload-to-cpu" not in live[2]
+ ):
+ card_free = _h3_card_free_bytes(state.device, state.gpu_ordinal)
+ free = None if card_free is None else card_free + file_bytes
+ else:
+ free = _h3_card_free_bytes(state.device, state.gpu_ordinal)
render_flags, render_resident = h3_native_render_flags(
runtime.offload_flags,
memory_mode = state.memory_mode,
- free_bytes = _h3_card_free_bytes(state.device, state.gpu_ordinal) if need else None,
+ free_bytes = free,
need_bytes = need,
)
if seed is None:
@@ -8978,31 +9144,42 @@ class VideoBackend:
"replaced by an install, so it is no longer the build this model "
"was checked against. Reload the model and try again."
)
- generated = runtime.engine.generate_video(
- runtime.files,
- SdCppVideoGenParams(
- prompt = prompt,
- width = width,
- height = height,
- num_frames = frames,
- fps = fps,
- steps = steps,
- cfg_scale = 1.0,
- seed = int(seed),
- init_img = init_img,
- end_img = end_img,
- ref_images = staged.images,
- ref_videos = staged.videos,
- ref_video_audios = staged.video_audios,
- ref_audios = staged.audios,
- flow_shift = flow_shift,
- ),
- output_path = str(output_path),
- offload = render_flags,
- env = dict(runtime.env) or None,
+ video_params = SdCppVideoGenParams(
+ prompt = prompt,
+ width = width,
+ height = height,
+ num_frames = frames,
+ fps = fps,
+ steps = steps,
+ cfg_scale = 1.0,
+ seed = int(seed),
+ init_img = init_img,
+ end_img = end_img,
+ ref_images = staged.images,
+ ref_videos = staged.videos,
+ ref_video_audios = staged.video_audios,
+ ref_audios = staged.audios,
+ flow_shift = flow_shift,
+ )
+ generated = self._h3_native_server_render(
+ runtime,
+ video_params,
+ output_path = output_path,
on_log = on_log,
- cancel_event = cancel,
+ cancel = cancel,
+ flags = render_flags,
+ env = dict(runtime.env),
)
+ if generated is None:
+ generated = runtime.engine.generate_video(
+ runtime.files,
+ video_params,
+ output_path = str(output_path),
+ offload = render_flags,
+ env = dict(runtime.env) or None,
+ on_log = on_log,
+ cancel_event = cancel,
+ )
except SdCppCancelled:
raise RuntimeError(VIDEO_CANCELLED_MSG) from None
except RuntimeError as exc:
@@ -9164,6 +9341,12 @@ class VideoBackend:
cannot observe a half-torn-down backend."""
state, self._state = self._state, None
if state is not None:
+ server_slot = getattr(getattr(state, "pipe", None), "server_slot", None)
+ if server_slot is not None:
+ try:
+ server_slot.stop("unload")
+ except Exception as exc: # noqa: BLE001 - teardown is best effort
+ logger.warning("video.h3_native_server_stop_failed: %s", exc)
# An in-flight background compile still runs the denoiser the teardown frees.
if getattr(state, "bg_compile", None) is not None:
try:
diff --git a/studio/backend/core/inference/video_minimax_h3.py b/studio/backend/core/inference/video_minimax_h3.py
index c1a7bcf050b..32d8d9ccf68 100644
--- a/studio/backend/core/inference/video_minimax_h3.py
+++ b/studio/backend/core/inference/video_minimax_h3.py
@@ -8,9 +8,10 @@ from __future__ import annotations
import math
import os
import tempfile
+import weakref
from dataclasses import dataclass
from pathlib import Path
-from typing import Any, Optional
+from typing import Any, Callable, Optional
# Must stay equal to the minimax-h3 family's `gguf_repo`. They are the same one-click pick, and main's
# test_curated_gguf_repos_are_unsloth_mirrors only checks the family field, so a divergence here would let that test
@@ -1326,6 +1327,329 @@ class MiniMaxH3NativeRuntime:
# The card the load resolved, kept for failure records: re-resolving at failure time can read None.
selected_card: Optional[str] = None
env: tuple[tuple[str, str], ...] = ()
+ # H3NativeServerSlot, or None for one-shot sd-cli only.
+ server_slot: Any = None
+
+
+# 0 / false / off: every render on a fresh one-shot sd-cli.
+H3_NATIVE_SERVER_ENV = "UNSLOTH_H3_NATIVE_SERVER"
+H3_NATIVE_SERVER_IDLE_ENV = "UNSLOTH_H3_NATIVE_SERVER_IDLE_S"
+# Covers watching a clip and editing the next prompt; an absent user gets the memory back within 3 minutes.
+H3_NATIVE_SERVER_IDLE_DEFAULT_S = 180.0
+# Free VRAM and available host RAM must each keep max(4 GiB, 15%) for an idle server to stay; same reserve as
+# diffusion_memory's _PIN_RESERVE_*.
+H3_NATIVE_SERVER_RESERVE_MIN_BYTES = 4 << 30
+H3_NATIVE_SERVER_RESERVE_FRACTION = 0.15
+
+
+def h3_native_server_enabled() -> bool:
+ value = os.environ.get(H3_NATIVE_SERVER_ENV, "").strip().lower()
+ return value not in ("0", "false", "no", "off")
+
+
+def h3_native_server_idle_s(env: Optional[dict] = None) -> float:
+ """The idle window in seconds; an unreadable or negative value falls back to the default."""
+ environ = env if env is not None else os.environ
+ raw = str(environ.get(H3_NATIVE_SERVER_IDLE_ENV, "") or "").strip()
+ if not raw:
+ return H3_NATIVE_SERVER_IDLE_DEFAULT_S
+ try:
+ value = float(raw)
+ except ValueError:
+ return H3_NATIVE_SERVER_IDLE_DEFAULT_S
+ if value != value or value < 0: # NaN or negative
+ return H3_NATIVE_SERVER_IDLE_DEFAULT_S
+ return value
+
+
+def h3_native_server_reserve_bytes(total_bytes: Optional[int]) -> int:
+ total = int(total_bytes or 0)
+ return max(H3_NATIVE_SERVER_RESERVE_MIN_BYTES, int(total * H3_NATIVE_SERVER_RESERVE_FRACTION))
+
+
+def h3_native_server_pressure(
+ *,
+ vram_free: Optional[int],
+ vram_total: Optional[int],
+ host_available: Optional[int],
+ host_total: Optional[int],
+) -> Optional[str]:
+ """Why an idle server must not be kept, or None. An unknown reading decides nothing for its side."""
+ if vram_free is not None and vram_total:
+ floor = h3_native_server_reserve_bytes(vram_total)
+ if int(vram_free) < floor:
+ return f"free VRAM {int(vram_free) >> 20} MiB below the {floor >> 20} MiB reserve"
+ if host_available is not None and host_total:
+ floor = h3_native_server_reserve_bytes(host_total)
+ if int(host_available) < floor:
+ return f"available host RAM {int(host_available) >> 20} MiB below the {floor >> 20} MiB reserve"
+ return None
+
+
+def h3_sibling_server_binary(cli_binary: Optional[str]) -> Optional[str]:
+ """The ``sd-server`` beside the vetted ``sd-cli``, or None. Not ``find_sd_server_binary``: that may resolve an
+ unvetted build from ``SD_SERVER_PATH`` / ``PATH``."""
+ if not cli_binary:
+ return None
+ cli = Path(cli_binary)
+ suffix = ".exe" if cli.suffix.lower() == ".exe" else ""
+ candidate = cli.with_name("sd-server" + suffix)
+ return str(candidate) if candidate.is_file() else None
+
+
+_LIVE_SLOTS: "weakref.WeakSet[H3NativeServerSlot]" = weakref.WeakSet()
+
+
+def release_h3_native_servers(reason: str) -> int:
+ """Stop every resident H3 sd-server (a busy one after its render). Returns how many; never raises."""
+ count = 0
+ for slot in list(_LIVE_SLOTS):
+ try:
+ if slot.release(reason):
+ count += 1
+ except Exception: # noqa: BLE001 -- a release must never break the caller that needs the memory
+ continue
+ return count
+
+
+class H3NativeServerSlot:
+ """At most one resident ``sd-server`` for a loaded MiniMax-H3 native runtime, so renders skip the ~32 GB reload.
+
+ Spawned with exactly the flags and env the one-shot sd-cli would get; another signature respawns it. Stopped by
+ the idle timer, ``pressure_probe`` after a render, ``release`` (busy: at render end) and ``stop`` on unload.
+ Registered as a managed-tree holder while alive so an install stands down.
+ """
+
+ def __init__(
+ self,
+ server_binary: str,
+ files: Any,
+ offload_flags: "tuple[str, ...] | list[str]" = (),
+ *,
+ pressure_probe: Optional[Callable[[], Optional[str]]] = None,
+ ) -> None:
+ import threading
+
+ self.server_binary = server_binary
+ self.files = files
+ self.offload_flags = tuple(offload_flags)
+ self.pressure_probe = pressure_probe
+ self.disabled_reason: Optional[str] = None
+ self.last_release_reason: Optional[str] = None
+ self._server: Any = None
+ self._stopping: Any = None
+ self._signature: Optional[tuple] = None
+ self._lock = threading.RLock()
+ self._busy = 0
+ self._release_pending: Optional[str] = None
+ self._timer: Any = None
+ # Bumped on every arm / cancel so a timer that fired as it was cancelled sees it is stale.
+ self._timer_token = 0
+ _LIVE_SLOTS.add(self)
+
+ def is_alive(self) -> bool:
+ # A server mid-stop still runs out of the managed tree.
+ return any(s is not None and s.is_alive() for s in (self._server, self._stopping))
+
+ @property
+ def signature(self) -> Optional[tuple]:
+ return self._signature
+
+ def _signature_for(self, flags: "tuple[str, ...]", env: "tuple[tuple[str, str], ...]") -> tuple:
+ files = self.files
+ file_key = tuple(
+ getattr(files, name, None)
+ for name in ("diffusion_model", "llm", "vae", "audio_vae", "llm_vision")
+ )
+ return (self.server_binary, file_key, tuple(flags), tuple(sorted(env)))
+
+ def alive_signature(self) -> Optional[tuple]:
+ with self._lock:
+ return self._signature if self.is_alive() else None
+
+ def get(
+ self,
+ flags: "Optional[tuple[str, ...] | list[str]]" = None,
+ env: "Optional[dict[str, str] | tuple[tuple[str, str], ...]]" = None,
+ *,
+ cancel_event: Any = None,
+ ) -> Any:
+ """The live server for this (flags, env), (re)spawning one if needed. ``cancel_event`` aborts a start: the
+ render holds the generate lock through the model load, so unload and Cancel would otherwise wait it out."""
+ import threading
+
+ from .sd_cpp_backend import register_tree_holder, unregister_tree_holder
+ from .sd_cpp_engine import SdCppCancelled, is_managed_binary
+ from .sd_cpp_server import SdCppServer
+
+ flag_tuple = tuple(self.offload_flags if flags is None else flags)
+ env_pairs = tuple((env or {}).items()) if isinstance(env, dict) else tuple(env or ())
+ wanted = self._signature_for(flag_tuple, env_pairs)
+ with self._lock:
+ self._cancel_timer_locked()
+ if self._server is not None and self._server.is_alive() and self._signature == wanted:
+ return self._server
+ if self._server is not None:
+ self._stop_locked(
+ "signature changed" if self._server.is_alive() else "server exited"
+ )
+ if cancel_event is not None and cancel_event.is_set():
+ raise SdCppCancelled("sd-server start was cancelled before launch.")
+ server = SdCppServer(self.server_binary)
+ managed = is_managed_binary(self.server_binary)
+ self._server = server
+ self._signature = wanted
+ if managed:
+ register_tree_holder(self)
+ started = threading.Event()
+ if cancel_event is not None:
+
+ def abort_on_cancel() -> None:
+ # stop() sets the abort before taking the lifecycle lock, so start() leaves its readiness wait.
+ while not started.is_set():
+ if cancel_event.wait(0.2):
+ if not started.is_set():
+ server.stop()
+ return
+
+ threading.Thread(
+ target = abort_on_cancel, daemon = True, name = "h3-sd-server-start-cancel"
+ ).start()
+ try:
+ server.start(
+ self.files,
+ offload = list(flag_tuple),
+ env = dict(env_pairs) or None,
+ # A context flag on sd-server (per-run on sd-cli); same RNG keeps seeds identical across paths.
+ extra_args = ["--rng", "cpu"],
+ )
+ except BaseException:
+ started.set()
+ self._server = None
+ self._signature = None
+ self._stopping = server
+ try:
+ server.stop()
+ finally:
+ self._stopping = None
+ unregister_tree_holder(self)
+ raise
+ started.set()
+ if cancel_event is not None and cancel_event.is_set() and not server.is_alive():
+ self._stop_locked("cancelled")
+ raise SdCppCancelled("sd-server start was cancelled.")
+ return server
+
+ def begin_render(self) -> None:
+ with self._lock:
+ self._busy += 1
+ self._cancel_timer_locked()
+
+ def end_render(self) -> Optional[str]:
+ """Stop the server (pending release, memory pressure, idle 0) or arm the idle timer. Returns the stop reason."""
+ with self._lock:
+ self._busy = max(0, self._busy - 1)
+ if self._busy or self._server is None:
+ return None
+ reason = self._release_pending
+ if reason is None and self.pressure_probe is not None:
+ try:
+ reason = self.pressure_probe()
+ except Exception: # noqa: BLE001 -- an unreadable probe keeps the idle timeout as the bound
+ reason = None
+ if reason:
+ reason = f"memory pressure: {reason}"
+ idle_s = h3_native_server_idle_s()
+ if reason is None and idle_s <= 0:
+ reason = "idle timeout 0"
+ with self._lock:
+ if self._busy or self._server is None:
+ return None
+ if reason is not None:
+ self._stop_locked(reason)
+ return reason
+ self._arm_timer_locked(idle_s)
+ return None
+
+ def release(self, reason: str) -> bool:
+ """Stop the server for another consumer; a busy one stops at ``end_render``. True when there was one.
+
+ Never waits on a server start (minutes under the lock, and the caller may hold the GPU arbiter): the release is
+ left pending for that render instead."""
+ if not self._lock.acquire(timeout = 0.5):
+ self._release_pending = reason
+ return True
+ try:
+ if self._server is None:
+ return False
+ if self._busy:
+ self._release_pending = reason
+ return True
+ self._stop_locked(reason)
+ return True
+ finally:
+ self._lock.release()
+
+ def stop(self, reason: str = "stopped") -> None:
+ with self._lock:
+ self._stop_locked(reason)
+
+ def disable(self, reason: str) -> None:
+ self.disabled_reason = reason
+ self.stop(reason)
+
+ def _stop_locked(self, reason: str) -> None:
+ from .sd_cpp_backend import unregister_tree_holder
+
+ self._cancel_timer_locked()
+ server, self._server = self._server, None
+ self._signature = None
+ self._release_pending = None
+ if server is None:
+ return
+ self._stopping = server
+ self.last_release_reason = reason
+ try:
+ import logging
+ logging.getLogger(__name__).info("h3 native sd-server stopped: %s", reason)
+ except Exception: # noqa: BLE001
+ pass
+ try:
+ server.stop()
+ finally:
+ self._stopping = None
+ unregister_tree_holder(self)
+
+ def _cancel_timer_locked(self) -> None:
+ self._timer_token += 1
+ timer, self._timer = self._timer, None
+ if timer is not None:
+ timer.cancel()
+
+ def _arm_timer_locked(self, idle_s: float) -> None:
+ import threading
+
+ self._cancel_timer_locked()
+ token = self._timer_token
+ ref = weakref.ref(self)
+
+ def fire() -> None:
+ slot = ref()
+ if slot is not None:
+ slot._on_idle(token)
+
+ timer = threading.Timer(idle_s, fire)
+ timer.daemon = True
+ timer.name = "h3-sd-server-idle"
+ self._timer = timer
+ timer.start()
+
+ def _on_idle(self, token: int) -> None:
+ with self._lock:
+ if token != self._timer_token or self._busy:
+ return
+ self._timer = None
+ self._stop_locked("idle timeout")
def transcode_video_to_mp4(source: Path, *, fps: int) -> bytes:
diff --git a/studio/backend/tests/test_h3_native_server.py b/studio/backend/tests/test_h3_native_server.py
new file mode 100644
index 00000000000..958c21ba93f
--- /dev/null
+++ b/studio/backend/tests/test_h3_native_server.py
@@ -0,0 +1,957 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""MiniMax-H3 native renders through a resident sd-server instead of one sd-cli per render.
+
+Hermetic: the server process and its HTTP client are faked; nothing spawns a binary or opens a
+socket. The real-binary check (pixels and audio identical to sd-cli for the same seed, keyframe
+and text-only) lives with the benchmark evidence, not here."""
+
+from __future__ import annotations
+
+import base64
+import sys
+import threading
+from pathlib import Path
+
+import pytest
+
+from core.inference import sd_cpp_backend
+from core.inference import sd_cpp_server as srv
+from core.inference.sd_cpp_args import (
+ SdCppModelFiles,
+ SdCppVideoGenParams,
+ build_sd_cpp_server_command,
+ build_vid_gen_request,
+ h3_server_eligible,
+)
+from core.inference.sd_cpp_engine import SdCppCancelled
+from core.inference.sd_cpp_server import SdCppServerUnsupported
+from core.inference.video import VideoBackend
+from core.inference import video_minimax_h3 as h3
+
+# Sibling imports need the tests dir on the path: pytest inserts rootdir, not this package.
+_TESTS_DIR = str(Path(__file__).resolve().parent)
+if _TESTS_DIR not in sys.path:
+ sys.path.insert(0, _TESTS_DIR)
+
+from test_sd_cpp_server import _FakeClient, _FakePopen, _Resp, _server_with # noqa: E402
+
+_FILES = SdCppModelFiles(
+ diffusion_model = "/m/h3.gguf",
+ vae = "/m/vae.safetensors",
+ audio_vae = "/m/audio_vae.safetensors",
+ llm = "/m/te.gguf",
+)
+
+
+def _params(**overrides) -> SdCppVideoGenParams:
+ values = dict(
+ prompt = "a dog on a beach",
+ width = 960,
+ height = 544,
+ num_frames = 124,
+ fps = 24,
+ steps = 20,
+ cfg_scale = 1.0,
+ seed = 42,
+ flow_shift = 12.0,
+ )
+ values.update(overrides)
+ return SdCppVideoGenParams(**values)
+
+
+def patched(monkeypatch):
+ monkeypatch.setattr(srv, "adopt_pid", lambda pid: None)
+ monkeypatch.setattr(srv, "forget_pid", lambda pid: None)
+ monkeypatch.setattr(srv, "child_popen_kwargs", lambda: {})
+ monkeypatch.setattr(srv, "windows_hidden_subprocess_kwargs", lambda: {})
+ return monkeypatch
+
+
+# -- request / argv -----------------------------------------------------------------------------
+
+
+def test_vid_gen_request_mirrors_the_cli_argv():
+ req = build_vid_gen_request(_params())
+ assert req == {
+ "prompt": "a dog on a beach",
+ "width": 960,
+ "height": 544,
+ "video_frames": 124,
+ "fps": 24,
+ "seed": 42,
+ "sample_params": {
+ "guidance": {"txt_cfg": 1.0},
+ "sample_steps": 20,
+ "flow_shift": 12.0,
+ },
+ # AVI: needs no WebM build, and is what the CUDA prebuilt sd-cli writes anyway.
+ "output_format": "avi",
+ "output_compression": 90,
+ }
+
+
+def test_vid_gen_request_carries_keyframes_and_rejects_mixing_with_references():
+ req = build_vid_gen_request(_params(), images_b64 = {"init_image": "AAA", "end_image": None})
+ assert req["init_image"] == "AAA"
+ assert "end_image" not in req
+ with pytest.raises(ValueError, match = "cannot be combined"):
+ build_vid_gen_request(_params(), images_b64 = {"init_image": "AAA"}, ref_images_b64 = ["BBB"])
+
+
+def test_reference_video_and_audio_stay_on_the_cli():
+ assert h3_server_eligible(_params()) is True
+ assert h3_server_eligible(_params(ref_images = ("/r.png",))) is True
+ for field in ("ref_videos", "ref_video_audios", "ref_audios"):
+ params = _params(**{field: ("/r",)})
+ assert h3_server_eligible(params) is False
+ with pytest.raises(ValueError):
+ build_vid_gen_request(params)
+
+
+def test_server_command_passes_the_audio_vae():
+ cmd = build_sd_cpp_server_command("/b/sd-server", _FILES, host = "127.0.0.1", port = 1)
+ assert cmd[cmd.index("--audio-vae") + 1] == "/m/audio_vae.safetensors"
+ no_audio = build_sd_cpp_server_command(
+ "/b/sd-server", SdCppModelFiles(diffusion_model = "/m/z.gguf"), host = "h", port = 1
+ )
+ assert "--audio-vae" not in no_audio
+
+
+# -- SdCppServer.vid_gen ------------------------------------------------------------------------
+
+
+def test_vid_gen_returns_container_bytes(patched):
+ blob = b"RIFF....AVI "
+ client = _FakeClient(
+ post = lambda url, json: _Resp(202, {"id": "jobV"}),
+ get = lambda url: _Resp(
+ 200,
+ {"status": "completed", "result": {"b64_json": base64.b64encode(blob).decode()}},
+ ),
+ )
+ s = _server_with(_FakePopen(), client)
+ assert s.vid_gen({"prompt": "x"}, poll_interval = 0.0) == blob
+ assert client.post_calls[0][0].endswith("/sdcpp/v1/vid_gen")
+
+
+ "status,text",
+ [(404, "Not Found"), (400, '{"error":"loaded model does not support vid_gen"}')],
+)
+def test_vid_gen_unsupported_is_its_own_error(patched, status, text):
+ s = _server_with(_FakePopen(), _FakeClient(post = lambda url, json: _Resp(status, text = text)))
+ with pytest.raises(SdCppServerUnsupported):
+ s.vid_gen({"prompt": "x"})
+
+
+def test_vid_gen_other_400_is_a_plain_failure(patched):
+ s = _server_with(
+ _FakePopen(), _FakeClient(post = lambda url, json: _Resp(400, text = "invalid parameters"))
+ )
+ with pytest.raises(RuntimeError) as info:
+ s.vid_gen({"prompt": "x"})
+ assert not isinstance(info.value, SdCppServerUnsupported)
+
+
+def test_vid_gen_failure_carries_the_log_cause(patched):
+ s = _server_with(
+ _FakePopen(),
+ _FakeClient(
+ post = lambda url, json: _Resp(202, {"id": "jobF"}),
+ get = lambda url: _Resp(
+ 200,
+ {
+ "status": "failed",
+ "error": {
+ "code": "generation_failed",
+ "message": "generate_video returned no results",
+ },
+ },
+ ),
+ ),
+ )
+ s._tail.append(
+ "ggml_backend_cuda_buffer_type_alloc_buffer: allocating 5463.50 MiB: out of memory"
+ )
+ with pytest.raises(RuntimeError, match = "out of memory"):
+ s.vid_gen({"prompt": "x"}, poll_interval = 0.0)
+
+
+# -- routing in the video backend ---------------------------------------------------------------
+
+
+class _FakeServer:
+ def __init__(
+ self,
+ *,
+ data = b"AVI",
+ exc = None,
+ alive_after = True,
+ ):
+ self.data = data
+ self.exc = exc
+ self.alive_after = alive_after
+ self.payloads = []
+ self._alive = True
+
+ def is_alive(self):
+ return self._alive
+
+ def vid_gen(
+ self,
+ payload,
+ *,
+ on_step = None,
+ cancel_event = None,
+ ):
+ self.payloads.append(payload)
+ if on_step is not None:
+ on_step("generate_video 960x544x124")
+ if self.exc is not None:
+ self._alive = self.alive_after
+ raise self.exc
+ return self.data
+
+
+class _FakeSlot:
+ def __init__(
+ self,
+ server = None,
+ start_exc = None,
+ ):
+ self.server = server
+ self.start_exc = start_exc
+ self.disabled_reason = None
+ self.stopped = 0
+ self.released = []
+ self.got = []
+ self.renders = 0
+ self.ended = 0
+
+ def get(
+ self,
+ flags = None,
+ env = None,
+ *,
+ cancel_event = None,
+ ):
+ self.got.append((flags, env))
+ if self.start_exc is not None:
+ raise self.start_exc
+ return self.server
+
+ def begin_render(self):
+ self.renders += 1
+
+ def end_render(self):
+ self.ended += 1
+
+ def release(self, reason):
+ self.released.append(reason)
+ return True
+
+ def alive_signature(self):
+ return None
+
+ def stop(self, reason = "stopped"):
+ self.stopped += 1
+
+ def disable(self, reason):
+ self.disabled_reason = reason
+ self.stop()
+
+
+class _Runtime:
+ def __init__(
+ self,
+ slot,
+ offload_flags = ("--offload-to-cpu",),
+ env = (),
+ ):
+ self.server_slot = slot
+ self.offload_flags = offload_flags
+ self.env = env
+
+
+def _render(
+ slot,
+ tmp_path,
+ params = None,
+ cancel = None,
+ **kwargs,
+):
+ return VideoBackend._h3_native_server_render(
+ None,
+ _Runtime(slot),
+ params or _params(),
+ output_path = tmp_path / "out.webm",
+ on_log = lambda line: None,
+ cancel = cancel or threading.Event(),
+ **kwargs,
+ )
+
+
+def test_server_render_writes_the_container(tmp_path, monkeypatch):
+ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)
+ server = _FakeServer(data = b"AVI-BYTES")
+ out = _render(_FakeSlot(server), tmp_path)
+ assert out == tmp_path / "out.webm"
+ assert out.read_bytes() == b"AVI-BYTES"
+ assert server.payloads[0]["seed"] == 42
+
+
+def test_keyframes_are_sent_inline(tmp_path, monkeypatch):
+ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)
+ first = tmp_path / "first.png"
+ first.write_bytes(b"PNG-FIRST")
+ server = _FakeServer()
+ _render(_FakeSlot(server), tmp_path, _params(init_img = str(first)))
+ assert base64.b64decode(server.payloads[0]["init_image"]) == b"PNG-FIRST"
+
+
+def test_no_slot_or_kill_switch_or_reference_video_uses_the_cli(tmp_path, monkeypatch):
+ assert (
+ VideoBackend._h3_native_server_render(
+ None,
+ _Runtime(None),
+ _params(),
+ output_path = tmp_path / "o",
+ on_log = print,
+ cancel = threading.Event(),
+ )
+ is None
+ )
+ slot = _FakeSlot(_FakeServer())
+ monkeypatch.setenv(h3.H3_NATIVE_SERVER_ENV, "0")
+ assert _render(slot, tmp_path) is None
+ assert slot.stopped == 1 # a server left from before the switch is released
+ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV)
+ server = _FakeServer()
+ slot = _FakeSlot(server)
+ assert _render(slot, tmp_path, _params(ref_videos = ("/frames",))) is None
+ assert server.payloads == []
+ # sd-cli reloads the whole model: a resident copy must not sit beside it.
+ assert slot.released and slot.got == []
+
+
+def test_start_failure_falls_back_and_stays_one_shot(tmp_path, monkeypatch):
+ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)
+ slot = _FakeSlot(start_exc = RuntimeError("sd-server failed to become ready"))
+ assert _render(slot, tmp_path) is None
+ assert "start failed" in slot.disabled_reason
+ # Disabled: the next render does not even try.
+ slot.start_exc = AssertionError("must not start again")
+ assert _render(slot, tmp_path) is None
+
+
+def test_unsupported_route_falls_back(tmp_path, monkeypatch):
+ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)
+ slot = _FakeSlot(_FakeServer(exc = SdCppServerUnsupported("no vid_gen")))
+ assert _render(slot, tmp_path) is None
+ assert slot.disabled_reason == "no vid_gen"
+
+
+def test_server_death_mid_render_falls_back(tmp_path, monkeypatch):
+ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)
+ slot = _FakeSlot(_FakeServer(exc = RuntimeError("connection lost"), alive_after = False))
+ assert _render(slot, tmp_path) is None
+ assert slot.disabled_reason.startswith("server died")
+
+
+def test_failed_job_on_a_live_server_raises_and_releases_it(tmp_path, monkeypatch):
+ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)
+ slot = _FakeSlot(_FakeServer(exc = RuntimeError("out of memory"), alive_after = True))
+ with pytest.raises(RuntimeError, match = "out of memory"):
+ _render(slot, tmp_path)
+ assert slot.stopped == 1
+ assert slot.disabled_reason is None # the next render respawns a server
+
+
+def test_cancel_is_a_cancellation_not_a_fallback(tmp_path, monkeypatch):
+ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)
+ cancel = threading.Event()
+ cancel.set()
+ slot = _FakeSlot(_FakeServer(exc = RuntimeError("server stopped"), alive_after = False))
+ with pytest.raises(SdCppCancelled):
+ _render(slot, tmp_path, cancel = cancel)
+ assert slot.disabled_reason is None
+
+
+# -- slot lifecycle -----------------------------------------------------------------------------
+
+
+def test_sibling_server_binary(tmp_path):
+ cli = tmp_path / "sd-cli"
+ cli.write_text("")
+ assert h3.h3_sibling_server_binary(str(cli)) is None
+ (tmp_path / "sd-server").write_text("")
+ assert h3.h3_sibling_server_binary(str(cli)) == str(tmp_path / "sd-server")
+ assert h3.h3_sibling_server_binary(None) is None
+
+
+def test_slot_holds_the_managed_tree_while_its_server_lives(monkeypatch):
+ started = []
+
+ class _Srv:
+ def __init__(self, binary):
+ self.binary = binary
+ self.alive = False
+ self.stopped = False
+
+ def start(
+ self,
+ files,
+ *,
+ offload = None,
+ env = None,
+ extra_args = None,
+ ):
+ started.append((files, offload, extra_args))
+ self.alive = True
+
+ def is_alive(self):
+ return self.alive
+
+ def stop(self):
+ self.alive = False
+ self.stopped = True
+
+ monkeypatch.setattr(srv, "SdCppServer", _Srv)
+ monkeypatch.setattr("core.inference.sd_cpp_engine.is_managed_binary", lambda b: True)
+ monkeypatch.setattr(sd_cpp_backend, "_sd_cpp_backend", None, raising = False)
+ slot = h3.H3NativeServerSlot(
+ "/managed/sd-server", _FILES, ("--offload-to-cpu", "--diffusion-fa")
+ )
+ assert sd_cpp_backend._managed_tree_in_use() is False
+ server = slot.get()
+ assert slot.get() is server # reused, not respawned
+ assert started == [(_FILES, ["--offload-to-cpu", "--diffusion-fa"], ["--rng", "cpu"])]
+ assert sd_cpp_backend._managed_tree_in_use() is True
+ with sd_cpp_backend._tree_claimed_for_install() as claimed:
+ assert claimed is False # an install stands down for the resident server
+ slot.stop()
+ assert server.stopped is True
+ assert sd_cpp_backend._managed_tree_in_use() is False
+
+
+def test_slot_respawns_a_dead_server(monkeypatch):
+ class _Srv:
+ def __init__(self, binary):
+ self.alive = False
+
+ def start(
+ self,
+ files,
+ *,
+ offload = None,
+ env = None,
+ extra_args = None,
+ ):
+ self.alive = True
+
+ def is_alive(self):
+ return self.alive
+
+ def stop(self):
+ self.alive = False
+
+ monkeypatch.setattr(srv, "SdCppServer", _Srv)
+ monkeypatch.setattr("core.inference.sd_cpp_engine.is_managed_binary", lambda b: False)
+ slot = h3.H3NativeServerSlot("/x/sd-server", _FILES, ())
+ first = slot.get()
+ first.alive = False # crashed while idle
+ second = slot.get()
+ assert second is not first and second.is_alive()
+ slot.stop()
+
+
+# -- spawn signature, idle timeout, pressure and release ------------------------------------------
+
+
+class _LifeSrv:
+ """A fake SdCppServer that records every spawn."""
+
+ spawns: list = []
+
+ def __init__(self, binary):
+ self.binary = binary
+ self.alive = False
+
+ def start(
+ self,
+ files,
+ *,
+ offload = None,
+ env = None,
+ extra_args = None,
+ ):
+ _LifeSrv.spawns.append({"offload": offload, "env": env, "extra_args": extra_args})
+ self.alive = True
+
+ def is_alive(self):
+ return self.alive
+
+ def stop(self):
+ self.alive = False
+
+
+def life(monkeypatch):
+ _LifeSrv.spawns = []
+ monkeypatch.setattr(srv, "SdCppServer", _LifeSrv)
+ monkeypatch.setattr("core.inference.sd_cpp_engine.is_managed_binary", lambda b: False)
+ monkeypatch.delenv(h3.H3_NATIVE_SERVER_IDLE_ENV, raising = False)
+ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)
+ slots = []
+
+ def make(**kwargs):
+ slot = h3.H3NativeServerSlot("/x/sd-server", _FILES, ("--offload-to-cpu",), **kwargs)
+ slots.append(slot)
+ return slot
+
+ yield make
+ for slot in slots:
+ slot.stop()
+
+
+RESIDENT = ["--diffusion-fa", "--backend", "diffusion=CUDA1,te=CUDA1,vae=CUDA1"]
+OFFLOAD = ["--offload-to-cpu", "--diffusion-fa", "--backend", "diffusion=CUDA1,te=CUDA1,vae=CUDA1"]
+SAGE_ENV = {h3.H3_QUANT_CUBLAS_ENV: h3.H3_QUANT_CUBLAS_MIN_BATCH}
+
+
+def test_slot_spawns_with_the_render_flags_and_env(life):
+ slot = life()
+ server = slot.get(RESIDENT + ["--sage-attn"], SAGE_ENV)
+ assert _LifeSrv.spawns == [
+ {"offload": RESIDENT + ["--sage-attn"], "env": SAGE_ENV, "extra_args": ["--rng", "cpu"]}
+ ]
+ # Same signature: reused, not respawned.
+ assert slot.get(RESIDENT + ["--sage-attn"], dict(SAGE_ENV)) is server
+ assert len(_LifeSrv.spawns) == 1
+ assert slot.alive_signature()[2] == tuple(RESIDENT + ["--sage-attn"])
+
+
+ "first, second",
+ [
+ ((RESIDENT, {}), (OFFLOAD, {})), # memory decision changed
+ ((RESIDENT, {}), (RESIDENT + ["--sage-attn"], SAGE_ENV)), # speed_mode=max switched on
+ ((RESIDENT + ["--sage-attn"], SAGE_ENV), (RESIDENT + ["--sage-attn"], {})), # env differs
+ ],
+)
+def test_slot_respawns_when_the_signature_changes(life, first, second):
+ slot = life()
+ old = slot.get(*first)
+ new = slot.get(*second)
+ assert new is not old
+ assert old.is_alive() is False # never two resident copies
+ assert [s["offload"] for s in _LifeSrv.spawns] == [first[0], second[0]]
+ assert slot.last_release_reason == "signature changed"
+
+
+def test_slot_signature_covers_the_model_files(life):
+ a = life()
+ b = h3.H3NativeServerSlot(
+ "/x/sd-server",
+ SdCppModelFiles(
+ diffusion_model = "/m/q8.gguf", vae = _FILES.vae, audio_vae = _FILES.audio_vae, llm = _FILES.llm
+ ),
+ )
+ try:
+ assert a._signature_for(tuple(RESIDENT), ()) != b._signature_for(tuple(RESIDENT), ())
+ finally:
+ b.stop()
+
+
+def test_idle_timeout_stops_the_server_and_leaves_no_timer(life, monkeypatch):
+ monkeypatch.setenv(h3.H3_NATIVE_SERVER_IDLE_ENV, "0.05")
+ slot = life()
+ slot.begin_render()
+ server = slot.get(RESIDENT, {})
+ assert slot.end_render() is None
+ timer = slot._timer
+ assert timer is not None and timer.daemon
+ timer.join(2.0)
+ assert not timer.is_alive()
+ assert server.is_alive() is False and slot.is_alive() is False
+ assert slot.last_release_reason == "idle timeout"
+ assert slot._timer is None
+
+
+def test_a_new_render_cancels_the_idle_timer(life, monkeypatch):
+ monkeypatch.setenv(h3.H3_NATIVE_SERVER_IDLE_ENV, "0.2")
+ slot = life()
+ slot.begin_render()
+ server = slot.get(RESIDENT, {})
+ slot.end_render()
+ first = slot._timer
+ slot.begin_render() # the next render starts before the window ends
+ assert slot._timer is None
+ first.join(1.0)
+ assert server.is_alive() is True
+ # A stale firing (the timer raced the cancel) is ignored too.
+ slot._on_idle(slot._timer_token - 1)
+ assert server.is_alive() is True
+ slot.end_render()
+ slot.stop()
+ assert slot._timer is None and server.is_alive() is False
+
+
+def test_stop_cancels_a_pending_timer(life, monkeypatch):
+ monkeypatch.setenv(h3.H3_NATIVE_SERVER_IDLE_ENV, "30")
+ slot = life()
+ slot.begin_render()
+ slot.get(RESIDENT, {})
+ slot.end_render()
+ timer = slot._timer
+ slot.stop("unload")
+ timer.join(1.0)
+ assert not timer.is_alive()
+
+
+def test_idle_zero_stops_right_after_the_render(life, monkeypatch):
+ monkeypatch.setenv(h3.H3_NATIVE_SERVER_IDLE_ENV, "0")
+ slot = life()
+ slot.begin_render()
+ server = slot.get(RESIDENT, {})
+ assert slot.end_render() == "idle timeout 0"
+ assert server.is_alive() is False
+
+
+ "raw, expected", [("", 180.0), ("45", 45.0), ("-1", 180.0), ("abc", 180.0), ("0", 0.0)]
+)
+def test_idle_env_parsing(raw, expected):
+ assert h3.h3_native_server_idle_s({h3.H3_NATIVE_SERVER_IDLE_ENV: raw}) == expected
+
+
+def test_memory_pressure_stops_the_server_after_the_render(life):
+ reasons = iter(["free VRAM 1 MiB below the 4096 MiB reserve"])
+ slot = life(pressure_probe = lambda: next(reasons))
+ slot.begin_render()
+ server = slot.get(RESIDENT, {})
+ assert slot.end_render().startswith("memory pressure: free VRAM")
+ assert server.is_alive() is False
+ assert slot._timer is None
+
+
+def test_a_broken_probe_keeps_the_idle_bound(life, monkeypatch):
+ monkeypatch.setenv(h3.H3_NATIVE_SERVER_IDLE_ENV, "30")
+
+ def boom():
+ raise OSError("nvidia-smi missing")
+
+ slot = life(pressure_probe = boom)
+ slot.begin_render()
+ server = slot.get(RESIDENT, {})
+ assert slot.end_render() is None
+ assert server.is_alive() is True and slot._timer is not None
+
+
+def test_pressure_thresholds():
+ gib = 1024**3
+ ok = dict(
+ vram_free = 40 * gib, vram_total = 80 * gib, host_available = 100 * gib, host_total = 200 * gib
+ )
+ assert h3.h3_native_server_pressure(**ok) is None
+ # 15% of an 80 GiB card is 12 GiB.
+ assert "VRAM" in h3.h3_native_server_pressure(**{**ok, "vram_free": 11 * gib})
+ # A 16 GiB card: the 4 GiB floor wins over 15%.
+ assert "VRAM" in h3.h3_native_server_pressure(
+ **{**ok, "vram_free": 3 * gib, "vram_total": 16 * gib}
+ )
+ assert "host RAM" in h3.h3_native_server_pressure(**{**ok, "host_available": 29 * gib})
+ # Unknown readings decide nothing.
+ assert (
+ h3.h3_native_server_pressure(
+ vram_free = None, vram_total = None, host_available = None, host_total = None
+ )
+ is None
+ )
+
+
+def test_release_stops_an_idle_server_and_defers_a_busy_one(life, monkeypatch):
+ monkeypatch.setenv(h3.H3_NATIVE_SERVER_IDLE_ENV, "30")
+ idle = life()
+ idle.begin_render()
+ idle_server = idle.get(RESIDENT, {})
+ idle.end_render()
+ busy = life()
+ busy.begin_render()
+ busy_server = busy.get(RESIDENT, {})
+ assert h3.release_h3_native_servers("export subprocess starting") >= 2
+ assert idle_server.is_alive() is False and idle._timer is None
+ assert idle.last_release_reason == "export subprocess starting"
+ # The render in flight finishes on its server; the server goes as it ends.
+ assert busy_server.is_alive() is True
+ assert busy.end_render() == "export subprocess starting"
+ assert busy_server.is_alive() is False
+
+
+def test_gpu_arbiter_releases_idle_video_servers_for_other_owners(monkeypatch):
+ from core.inference import gpu_arbiter
+
+ seen = []
+ monkeypatch.setattr(h3, "release_h3_native_servers", lambda reason: seen.append(reason) or 0)
+ monkeypatch.setattr(gpu_arbiter, "_owner", None)
+ monkeypatch.setattr(gpu_arbiter, "raise_if_other_accounts_active", lambda *a, **k: None)
+ gpu_arbiter.acquire_for(gpu_arbiter.CHAT, account_id = "a")
+ assert seen == ["GPU acquired for chat"]
+ gpu_arbiter.acquire_for(gpu_arbiter.VIDEO, account_id = "a", allow_evict = True)
+ assert seen == ["GPU acquired for chat"] # the video owner itself keeps its server
+
+
+def test_render_hands_the_cli_flags_and_env_to_the_slot(tmp_path, monkeypatch):
+ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)
+ slot = _FakeSlot(_FakeServer())
+ _render(slot, tmp_path, flags = RESIDENT + ["--sage-attn"], env = SAGE_ENV)
+ assert slot.got == [(RESIDENT + ["--sage-attn"], SAGE_ENV)]
+ assert slot.renders == 1 and slot.ended == 1
+
+
+def test_render_ends_the_slot_render_on_failure_too(tmp_path, monkeypatch):
+ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)
+ slot = _FakeSlot(_FakeServer(exc = RuntimeError("out of memory"), alive_after = True))
+ with pytest.raises(RuntimeError):
+ _render(slot, tmp_path)
+ assert slot.ended == 1
+
+
+def test_kill_switch_stops_a_live_server_and_uses_the_cli(tmp_path, monkeypatch, life):
+ slot = life()
+ slot.begin_render()
+ server = slot.get(RESIDENT, {})
+ slot.end_render()
+ monkeypatch.setenv(h3.H3_NATIVE_SERVER_ENV, "0")
+ assert _render(slot, tmp_path) is None
+ assert server.is_alive() is False and slot._timer is None
+
+
+# -- through VideoBackend.generate ----------------------------------------------------------------
+
+
+def _generate_backend(
+ monkeypatch,
+ tmp_path,
+ slot,
+ *,
+ env = (),
+):
+ import dataclasses
+
+ from test_h3_native_resident import AUTO_FLAGS, _backend_with_files
+
+ calls: list = []
+ backend = _backend_with_files(
+ monkeypatch, tmp_path, calls, memory_mode = "auto", flags = AUTO_FLAGS
+ )
+ backend._state = dataclasses.replace(
+ backend._state,
+ pipe = dataclasses.replace(backend._state.pipe, server_slot = slot, env = tuple(env)),
+ )
+ return backend, calls, AUTO_FLAGS
+
+
+class _StartFailsSlot(_FakeSlot):
+ def __init__(self, live = None):
+ super().__init__(start_exc = RuntimeError("no server"))
+ self.live = live
+
+ def alive_signature(self):
+ return self.live
+
+
+def test_generate_gives_the_server_and_the_fallback_cli_the_same_flags_and_env(
+ monkeypatch, tmp_path
+):
+ import core.inference.video as video
+
+ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)
+ gib = 1024**3
+ monkeypatch.setattr(video, "_h3_card_free_bytes", lambda device, ordinal: 90 * gib)
+ slot = _StartFailsSlot()
+ backend, calls, auto = _generate_backend(
+ monkeypatch, tmp_path, slot, env = tuple(SAGE_ENV.items())
+ )
+ backend.generate(prompt = "a fox", width = 960, height = 544)
+ resident = [f for f in auto if f not in ("--offload-to-cpu", "--stream-layers")]
+ assert slot.got == [(resident, SAGE_ENV)]
+ assert calls[0]["offload"] == resident and calls[0]["env"] == SAGE_ENV
+
+
+def test_a_live_resident_server_is_not_pushed_back_to_offload_by_its_own_usage(
+ monkeypatch, tmp_path
+):
+ import core.inference.video as video
+
+ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)
+ from test_h3_native_resident import AUTO_FLAGS
+
+ gib = 1024**3
+ resident = tuple(f for f in AUTO_FLAGS if f not in ("--offload-to-cpu", "--stream-layers"))
+ slot = _StartFailsSlot(live = ("/x/sd-server", (), resident, ()))
+ # The live server holds the 32 GiB bundle, so the card reads only 10 GiB free: enough for this clip's activations.
+ monkeypatch.setattr(video, "_h3_card_free_bytes", lambda *_a: 10 * gib)
+ backend, calls, _ = _generate_backend(monkeypatch, tmp_path, slot)
+ result = backend.generate(prompt = "a fox", width = 960, height = 544)
+ assert slot.got[0][0] == list(resident)
+ assert result["offload_policy"] == "none"
+
+
+def test_a_live_resident_server_does_not_hide_a_clip_too_large_for_the_card(monkeypatch, tmp_path):
+ import core.inference.video as video
+
+ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)
+ from test_h3_native_resident import AUTO_FLAGS
+
+ gib = 1024**3
+ resident = tuple(f for f in AUTO_FLAGS if f not in ("--offload-to-cpu", "--stream-layers"))
+ slot = _StartFailsSlot(live = ("/x/sd-server", (), resident, ()))
+ monkeypatch.setattr(video, "_h3_card_free_bytes", lambda *_a: 10 * gib)
+ backend, calls, _ = _generate_backend(monkeypatch, tmp_path, slot)
+ # 8x the pixel volume: ~43 GiB of activations on top of the weights, more than the 10 GiB the card has left.
+ result = backend.generate(prompt = "a fox", width = 1920, height = 1088, num_frames = 241)
+ assert "--offload-to-cpu" in slot.got[0][0]
+ assert result["offload_policy"] == "group"
+
+
+def test_release_does_not_wait_on_a_server_start(life, monkeypatch):
+ monkeypatch.setenv(h3.H3_NATIVE_SERVER_IDLE_ENV, "30")
+ slot = life()
+ slot.begin_render()
+ server = slot.get(RESIDENT, {})
+ holder_ready, done = threading.Event(), threading.Event()
+
+ def hold(): # stands in for a render spawning its server under the slot lock
+ with slot._lock:
+ holder_ready.set()
+ done.wait(5)
+
+ th = threading.Thread(target = hold)
+ th.start()
+ holder_ready.wait(2)
+ import time as _time
+
+ t0 = _time.monotonic()
+ assert slot.release("chat load") is True
+ assert _time.monotonic() - t0 < 2.0
+ done.set()
+ th.join(2)
+ # The pending release is honoured when that render ends.
+ assert slot.end_render() == "chat load"
+ assert server.is_alive() is False
+
+
+def test_cancel_aborts_a_server_start_in_progress(monkeypatch):
+ """Unload and Cancel wait on the render, which holds the generate lock through the server's model load."""
+ import time as _time
+
+ monkeypatch.setattr("core.inference.sd_cpp_engine.is_managed_binary", lambda b: False)
+ loading = threading.Event()
+
+ class _SlowSrv:
+ def __init__(self, binary):
+ self.aborted = threading.Event()
+ self.stops = 0
+
+ def start(
+ self,
+ files,
+ *,
+ offload = None,
+ env = None,
+ extra_args = None,
+ ):
+ loading.set()
+ if not self.aborted.wait(10):
+ raise AssertionError("start was never aborted")
+ raise SdCppCancelled("sd-server startup was cancelled.")
+
+ def stop(self):
+ self.stops += 1
+ self.aborted.set()
+
+ def is_alive(self):
+ return False
+
+ monkeypatch.setattr(srv, "SdCppServer", _SlowSrv)
+ slot = h3.H3NativeServerSlot("/x/sd-server", _FILES)
+ cancel = threading.Event()
+ threading.Timer(0.3, cancel.set).start()
+ t0 = _time.monotonic()
+ with pytest.raises(SdCppCancelled):
+ slot.get(RESIDENT, {}, cancel_event = cancel)
+ assert loading.is_set() and _time.monotonic() - t0 < 5
+ assert slot.alive_signature() is None and slot.disabled_reason is None
+ # Already cancelled: nothing is spawned at all.
+ with pytest.raises(SdCppCancelled):
+ slot.get(RESIDENT, {}, cancel_event = cancel)
+
+
+def test_a_live_resident_server_keeps_the_committed_offload_when_the_card_is_unreadable(
+ monkeypatch, tmp_path
+):
+ import core.inference.video as video
+
+ monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)
+ from test_h3_native_resident import AUTO_FLAGS
+
+ resident = tuple(f for f in AUTO_FLAGS if f not in ("--offload-to-cpu", "--stream-layers"))
+ slot = _StartFailsSlot(live = ("/x/sd-server", (), resident, ()))
+ monkeypatch.setattr(video, "_h3_card_free_bytes", lambda *_a: None)
+ backend, calls, _ = _generate_backend(monkeypatch, tmp_path, slot)
+ backend.generate(prompt = "a fox", width = 960, height = 544)
+ assert slot.got[0][0] == list(AUTO_FLAGS)
+
+
+def test_host_reserve_is_a_share_of_the_cgroup_limit(monkeypatch):
+ import core.inference.diffusion_memory as dm
+ import core.inference.video as video
+
+ gib_mib = 1024
+ monkeypatch.setattr(video, "_h3_card_memory_bytes", lambda *_a: (None, None))
+ # A 32 GiB container on a 512 GiB host with 20 GiB still chargeable: 15% of 32 GiB is under 20 GiB.
+ monkeypatch.setattr(dm, "_system_memory_mib", lambda: (512 * gib_mib, 400 * gib_mib))
+ monkeypatch.setattr(dm, "_available_system_memory_mib", lambda: 20 * gib_mib)
+ monkeypatch.setattr(dm, "_cgroup_memory_limit_mib", lambda: 32 * gib_mib)
+ assert video._h3_native_server_pressure("cuda", None) is None
+ # Without a limit the physical total stands.
+ monkeypatch.setattr(dm, "_cgroup_memory_limit_mib", lambda: None)
+ assert "host RAM" in video._h3_native_server_pressure("cuda", None)
+
+
+def test_the_tree_stays_held_until_the_server_has_exited(monkeypatch):
+ monkeypatch.setattr("core.inference.sd_cpp_engine.is_managed_binary", lambda b: True)
+ seen = []
+
+ class _ExitingSrv:
+ def __init__(self, binary):
+ self.alive = False
+
+ def start(
+ self,
+ files,
+ *,
+ offload = None,
+ env = None,
+ extra_args = None,
+ ):
+ self.alive = True
+
+ def is_alive(self):
+ return self.alive
+
+ def stop(self):
+ # SIGTERM sent, process not reaped yet: an install must still stand down.
+ seen.append(sd_cpp_backend._external_tree_holder_alive())
+ self.alive = False
+
+ monkeypatch.setattr(srv, "SdCppServer", _ExitingSrv)
+ slot = h3.H3NativeServerSlot("/x/sd-server", _FILES)
+ slot.get(RESIDENT, {})
+ slot.release("chat load")
+ assert seen == [True]
+ assert sd_cpp_backend._external_tree_holder_alive() is False