Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: settle device memory once per video plan, wake sd.cpp polls on server output (#16797)

Daniel Han416cedd19d73
diff --git a/studio/backend/core/inference/sd_cpp_server.py b/studio/backend/core/inference/sd_cpp_server.py
index b4b94dbf8e8..b31718fa97b 100644
--- a/studio/backend/core/inference/sd_cpp_server.py
+++ b/studio/backend/core/inference/sd_cpp_server.py
@@ -73,6 +73,47 @@ _TERMINAL_CANCELLED = "cancelled"
_CANCEL_GRACE_S = 5.0
+# Floor between a log line and the next poll; caps log-driven polling at 50/s.
+_LOG_WAKE_FLOOR_S = 0.02
+
+
+class _LogPacedWait:
+ """Sleeps ``interval`` between polls; a server log line cuts it short and starts polls at ``floor``,
+ doubling back to ``interval``. sd-server logs just before a status flip or port bind, not at it.
+ HTTP status stays the only source of truth."""
+
+ def __init__(
+ self,
+ wake: Optional[threading.Event],
+ interval: float,
+ floor: float = _LOG_WAKE_FLOOR_S,
+ ) -> None:
+ self._wake = wake
+ self._interval = max(0.0, float(interval))
+ self._floor = max(0.0, min(float(floor), self._interval))
+ self._follow: Optional[float] = None
+ if wake is not None:
+ wake.clear()
+
+ def wait(self) -> None:
+ if self._wake is None or self._floor <= 0.0:
+ time.sleep(self._interval)
+ return
+ if self._follow is None:
+ woke = self._wake.wait(self._interval)
+ else:
+ woke = self._wake.wait(self._follow)
+ if woke:
+ self._wake.clear()
+ time.sleep(self._floor)
+ self._wake.clear()
+ self._follow = self._floor
+ return
+ if self._follow is not None:
+ nxt = self._follow * 2.0
+ self._follow = nxt if nxt < self._interval else None
+
+
def _socket_owner_is_always_named() -> bool:
"""Windows names every socket's owner: an unnamed one there is a psutil quirk (#15022)."""
return sys.platform == "win32"
@@ -132,6 +173,8 @@ class SdCppServer:
self._abort = threading.Event()
# Set during a generation so the stdout drain feeds the step-progress callback.
self._step_listener: Optional[Callable[[str], None]] = None
+ # Set by the stdout drain on every line (and at EOF) so readiness / job polls run sooner.
+ self._log_wake = threading.Event()
# trust_env=False: loopback-only client must not route through HTTP(S)_PROXY.
self._client = httpx.Client(timeout = 30.0, trust_env = False)
self._scratch_dir: Optional[str] = None
@@ -290,6 +333,7 @@ class SdCppServer:
"""A 200 is a true ready signal: upstream binds the port only after the model has loaded."""
deadline = time.monotonic() + timeout
url = f"{self.base_url}{_READY_PATH}"
+ pacer = _LogPacedWait(getattr(self, "_log_wake", None), interval)
while time.monotonic() < deadline:
# stop() sets _abort so the wait bails early instead of holding the load for startup_timeout.
if self._abort.is_set():
@@ -306,7 +350,7 @@ class SdCppServer:
return True
except (*_TRANSPORT_ERRORS, httpx.TimeoutException):
pass
- time.sleep(interval)
+ pacer.wait()
logger.error("sd-server readiness timed out after %ss", timeout)
return False
@@ -412,8 +456,16 @@ class SdCppServer:
cb(line)
except Exception: # noqa: BLE001 -- a progress callback must never break drain
pass
+ # After the callback, so a woken poll never outruns this line's progress event.
+ self._wake_pollers()
except Exception: # noqa: BLE001 -- drain thread must never raise (pipe closed at teardown)
pass
+ self._wake_pollers()
+
+ def _wake_pollers(self) -> None:
+ wake = getattr(self, "_log_wake", None)
+ if wake is not None:
+ wake.set()
def reclaimable_params_vram_gb(
self, physical_gpu_id: Optional[int]
@@ -549,6 +601,8 @@ class SdCppServer:
self._step_listener = on_step
job_id: Optional[str] = None
+ # Before submit, so a line logged while the POST is in flight already counts.
+ pacer = _LogPacedWait(getattr(self, "_log_wake", None), poll_interval)
try:
try:
resp = self._client.post(
@@ -642,7 +696,7 @@ class SdCppServer:
raise RuntimeError(message)
if status == _TERMINAL_CANCELLED:
raise SdCppCancelled("sd-server generation was cancelled.")
- time.sleep(poll_interval)
+ pacer.wait()
finally:
self._step_listener = None
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 1d6be78e675..2e78f733486 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -3928,11 +3928,13 @@ def _video_seed_stays_resident(
reclaimable_mib: int = 0,
hf_token: Optional[str] = None,
ltx25_dev: bool = False,
+ device_memory_reader: Optional[Callable[[], Any]] = None,
) -> bool:
"""True when an artifact-sized plan keeps the denoiser resident (text encoders may stream).
``scheme=None`` sizes the released bf16 DiT instead, and then also requires a measured budget (see
- ``_auto_keeps_resident_bf16``)."""
+ ``_auto_keeps_resident_bf16``). ``device_memory_reader`` lets one planner call settle the card once
+ for both checks; it must return the raw settled snapshot (reclaimable is applied here)."""
from .diffusion_component_sizes import video_components_gb
components = video_components_gb(fam, base_repo, allow_hub = True, token = hf_token, dev = ltx25_dev)
@@ -3974,7 +3976,11 @@ def _video_seed_stays_resident(
from .video_ltx25 import LTX25_WORKER_RESIDENT_GB
companions_gb += LTX25_WORKER_RESIDENT_GB
model_dense_mib = int((denoiser_gb + companions_gb) * mib_per_gb)
- device_memory = settled_snapshot_device_memory(target)
+ device_memory = (
+ settled_snapshot_device_memory(target)
+ if device_memory_reader is None
+ else device_memory_reader()
+ )
if reclaimable_mib > 0 and device_memory.free_mib is not None:
free = device_memory.free_mib + int(reclaimable_mib)
if device_memory.total_mib is not None:
@@ -7061,6 +7067,14 @@ class VideoBackend:
getattr(resident, "pipe", None),
ordinal = _target_ordinal(target),
)
+ settled: list[Any] = []
+
+ def _settled_once() -> Any:
+ # Both checks plan on one settled read: each settle can sleep up to 2 s.
+ if not settled:
+ settled.append(settled_snapshot_device_memory(target))
+ return settled[0]
+
if _auto_seed_yields_to_resident_bf16(
scheme, transformer_quant, fam
) and _video_seed_stays_resident(
@@ -7073,6 +7087,7 @@ class VideoBackend:
reclaimable_mib = reclaimable,
hf_token = hf_token,
ltx25_dev = ltx25_dev,
+ device_memory_reader = _settled_once,
):
# auto keeps a resident bf16 DiT, so the dense shards are what this load runs.
return None
@@ -7083,12 +7098,10 @@ class VideoBackend:
memory_mode = memory_mode,
text_encoder_quant = text_encoder_quant,
base_repo = base,
- reclaimable_mib = _pipeline_device_mib(
- getattr(resident, "pipe", None),
- ordinal = _target_ordinal(target),
- ),
+ reclaimable_mib = reclaimable,
hf_token = hf_token,
ltx25_dev = ltx25_dev,
+ device_memory_reader = _settled_once,
):
logger.info(
"video.denoiser_prequant: an artifact-sized plan for %s still offloads on "
diff --git a/studio/backend/tests/test_sd_cpp_server_log_wake.py b/studio/backend/tests/test_sd_cpp_server_log_wake.py
new file mode 100644
index 00000000000..4d0801ef7cb
--- /dev/null
+++ b/studio/backend/tests/test_sd_cpp_server_log_wake.py
@@ -0,0 +1,251 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""sd-server polls wake on server log lines: a real loopback HTTP fake plus a piped stdout.
+
+Upstream sd-server (examples/server/async_jobs.cpp) logs ``generate_image completed`` and then encodes
+the PNG before the job flips to ``completed``; ``listening on`` is logged just before the port binds.
+Neither flip has its own line, so the poll must follow a line up, not trust it.
+"""
+
+from __future__ import annotations
+
+import base64
+import io
+import json
+import os
+import socket
+import statistics
+import threading
+import time
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+import pytest
+
+from core.inference import sd_cpp_server as srv
+from core.inference.sd_cpp_server import SdCppServer
+
+_PNG = base64.b64decode(
+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
+)
+
+
+class _PipeProc:
+ """A live child whose stdout is a real pipe the test writes server log lines into."""
+
+ def __init__(self):
+ r, w = os.pipe()
+ self.stdout = io.TextIOWrapper(io.FileIO(r, "r"), encoding = "utf-8", errors = "replace")
+ self._w = io.FileIO(w, "w")
+ self.pid = 2**31 - 1 # never a live process
+ self.returncode = None
+
+ def log(self, line: str) -> None:
+ self._w.write((line + "\n").encode())
+
+ def poll(self):
+ return self.returncode
+
+ def terminate(self):
+ self.returncode = 0
+ try:
+ self._w.close()
+ except OSError:
+ pass
+
+ def kill(self):
+ self.terminate()
+
+ def wait(self, timeout = None):
+ return self.returncode
+
+
+class _FakeSdServer:
+ """Loopback job API: one img_gen job that flips to completed ``gap`` after its last log line."""
+
+ def __init__(
+ self,
+ proc: _PipeProc,
+ *,
+ step_lines = 3,
+ step_s = 0.07,
+ gap_s = 0.03,
+ ):
+ self.proc = proc
+ self.step_lines, self.step_s, self.gap_s = step_lines, step_s, gap_s
+ self.requests: list[tuple[str, str, bytes]] = []
+ self.status = "queued"
+ self.flipped_at: float | None = None
+ fake = self
+
+ class _H(BaseHTTPRequestHandler):
+ def log_message(self, *a):
+ pass
+
+ def _send(self, code, payload):
+ body = json.dumps(payload).encode()
+ self.send_response(code)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+
+ def do_POST(self):
+ body = self.rfile.read(int(self.headers.get("Content-Length") or 0))
+ fake.requests.append(("POST", self.path, body))
+ threading.Thread(target = fake._run_job, daemon = True).start()
+ self._send(202, {"id": "job1", "status": "queued"})
+
+ def do_GET(self):
+ fake.requests.append(("GET", self.path, b""))
+ if self.path == "/v1/models":
+ return self._send(200, {"data": []})
+ if fake.status != "completed":
+ return self._send(200, {"id": "job1", "status": fake.status, "result": None})
+ b64 = base64.b64encode(_PNG).decode()
+ self._send(
+ 200,
+ {
+ "id": "job1",
+ "status": "completed",
+ "result": {"images": [{"index": 0, "b64_json": b64}]},
+ },
+ )
+
+ self._handler = _H
+ self.httpd: ThreadingHTTPServer | None = None
+
+ def bind(self, port: int) -> None:
+ self.httpd = ThreadingHTTPServer(("127.0.0.1", port), self._handler)
+ threading.Thread(target = self.httpd.serve_forever, daemon = True).start()
+
+ def _run_job(self):
+ self.status = "generating"
+ for i in range(self.step_lines):
+ time.sleep(self.step_s)
+ self.proc.log(f"|=====| {i + 1}/{self.step_lines} - 14.2it/s")
+ self.proc.log("generate_image completed in 0.25s")
+ time.sleep(self.gap_s) # PNG encode + base64 before the status flip, with no line
+ self.flipped_at = time.monotonic()
+ self.status = "completed"
+
+ def close(self):
+ if self.httpd is not None:
+ self.httpd.shutdown()
+ self.httpd.server_close()
+
+
+def _free_port() -> int:
+ with socket.socket() as s:
+ s.bind(("127.0.0.1", 0))
+ return s.getsockname()[1]
+
+
+def _attach(proc: _PipeProc, port: int) -> SdCppServer:
+ s = SdCppServer("sd-server")
+ s._process = proc
+ s.port = port
+ s._stdout_thread = threading.Thread(target = s._drain_stdout, args = (proc,), daemon = True)
+ s._stdout_thread.start()
+ return s
+
+
+def _one_job(on_step):
+ proc = _PipeProc()
+ fake = _FakeSdServer(proc)
+ port = _free_port()
+ fake.bind(port)
+ s = _attach(proc, port)
+ try:
+ images = s.img_gen({"prompt": "a cat", "seed": 7}, on_step = on_step)
+ done = time.monotonic()
+ return done - fake.flipped_at, images, fake.requests
+ finally:
+ s.stop()
+ fake.close()
+
+
+def test_job_completion_is_seen_soon_after_the_last_log_line():
+ lags, steps = [], []
+ for _ in range(5):
+ lag, images, requests = _one_job(steps.append)
+ lags.append(lag)
+ assert images == [_PNG] # same bytes as served
+ posts = [r for r in requests if r[0] == "POST"]
+ assert [(m, p, json.loads(b)) for m, p, b in posts] == [
+ ("POST", "/sdcpp/v1/img_gen", {"prompt": "a cat", "seed": 7})
+ ]
+ assert all(p == "/sdcpp/v1/jobs/job1" for m, p, _ in requests if m == "GET")
+ # Log-driven polling stays bounded: at most one GET per floor over the job.
+ assert len(requests) < 60
+ # Every server line reached the progress callback, in order, before the result.
+ assert steps == [
+ line
+ for _ in range(5)
+ for line in (
+ "|=====| 1/3 - 14.2it/s",
+ "|=====| 2/3 - 14.2it/s",
+ "|=====| 3/3 - 14.2it/s",
+ "generate_image completed in 0.25s",
+ )
+ ]
+ median = statistics.median(lags)
+ print(
+ f"job flip -> img_gen return lag: median {median * 1000:.1f} ms, all {[round(x * 1000) for x in lags]}"
+ )
+ assert median < 0.1, lags
+
+
+def test_ready_is_seen_soon_after_listening_line(monkeypatch):
+ monkeypatch.setattr(SdCppServer, "_port_is_ours", lambda self: True)
+ lags = []
+ for _ in range(5):
+ proc = _PipeProc()
+ fake = _FakeSdServer(proc)
+ port = _free_port()
+ s = _attach(proc, port)
+ bound: list[float] = []
+
+ def _serve():
+ time.sleep(0.12) # model load
+ proc.log(f"listening on: http://127.0.0.1:{port}")
+ time.sleep(0.005)
+ # Before bind: a ready poll can return before this thread runs again.
+ bound.append(time.monotonic())
+ fake.bind(port)
+
+ threading.Thread(target = _serve, daemon = True).start()
+ try:
+ assert s._wait_ready(10.0)
+ lags.append(time.monotonic() - bound[0])
+ finally:
+ s.stop()
+ fake.close()
+ median = statistics.median(lags)
+ print(
+ f"bind -> ready lag: median {median * 1000:.1f} ms, all {[round(x * 1000) for x in lags]}"
+ )
+ # The line comes just before the bind, so the first probe can hit a closed port; Windows takes
+ # a few hundred ms to refuse a loopback connect, so there it only has to beat the old 0.5 s poll.
+ assert median < (0.5 if os.name == "nt" else 0.1), lags
+
+
+def test_timer_still_drives_polls_without_log_lines():
+ """No line at all: the fallback timer alone finishes the job (pre-change behaviour)."""
+ wait = srv._LogPacedWait(threading.Event(), 0.05)
+ t0 = time.monotonic()
+ wait.wait()
+ assert 0.04 <= time.monotonic() - t0 < 0.5
+
+
+def test_follow_up_backs_off_to_the_timer():
+ wake = threading.Event()
+ wait = srv._LogPacedWait(wake, 0.4, floor = 0.02)
+ wake.set()
+ wait.wait() # line: one floor
+ follows = []
+ for _ in range(6):
+ follows.append(wait._follow)
+ wait.wait()
+ assert follows[:4] == [0.02, 0.04, 0.08, 0.16]
+ assert follows[5] is None # back on the plain interval timer
diff --git a/studio/backend/tests/test_video_seed_settle_once.py b/studio/backend/tests/test_video_seed_settle_once.py
new file mode 100644
index 00000000000..69bd643da7f
--- /dev/null
+++ b/studio/backend/tests/test_video_seed_settle_once.py
@@ -0,0 +1,151 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The pre-download auto denoiser planner settles the card once for both of its seed checks."""
+
+import itertools
+import time
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+import core.inference.diffusion_component_sizes as dcs
+import core.inference.diffusion_memory as dm
+import core.inference.video as vid
+from core.inference.diffusion_memory import DeviceMemory
+from core.inference.video_families import detect_video_family
+
+MIB_PER_GIB = 1024
+WAN5B = "Wan-AI/Wan2.2-TI2V-5B-Diffusers"
+
+
+class _Card:
+ """Fake device-memory provider under the real settle loop: counts reads and settle sleeps."""
+
+ def __init__(self, monkeypatch, free_mib, total_mib):
+ self.reads = 0
+ self.sleeps = 0
+ self.free_mib = free_mib
+ self.total_mib = total_mib
+ monkeypatch.setattr(dm, "snapshot_device_memory", self._read)
+ monkeypatch.setattr(time, "sleep", self._sleep)
+
+ def _read(self, _target):
+ self.reads += 1
+ return DeviceMemory("cuda", "cuda", "discrete_vram", self.free_mib, self.total_mib)
+
+ def _sleep(self, _s):
+ self.sleeps += 1
+
+
+def _backend(monkeypatch, *, scheme, resident_mib):
+ fam = detect_video_family(WAN5B)
+ backend = vid.VideoBackend()
+ resident_pipe = object() if resident_mib else None
+ backend._state = types.SimpleNamespace(pipe = resident_pipe)
+ monkeypatch.setattr(vid, "_video_auto_denoiser_scheme", lambda fam, **kw: scheme)
+ monkeypatch.setattr(
+ vid,
+ "_pipeline_device_mib",
+ lambda pipe, ordinal = None: resident_mib if pipe is resident_pipe and pipe else 0,
+ )
+ monkeypatch.setattr(
+ vid,
+ "resolve_diffusion_device_target",
+ lambda **_k: types.SimpleNamespace(
+ device = "cuda", dtype = torch.bfloat16, ordinal = None, supports_model_cpu_offload = True
+ ),
+ )
+ monkeypatch.setattr(
+ dcs, "video_components_gb", lambda fam, base, **kw: tuple(fam.bf16_components_gb)
+ )
+ return backend, fam
+
+
+def _plan(backend, fam, requested):
+ return backend._video_planned_auto_denoiser_scheme(
+ fam, base = None, kind = "pipeline", transformer_quant = requested, speed_mode = None
+ )
+
+
+def _main_arm(monkeypatch):
+ """The pre-fix planner: every seed check runs its own settle."""
+ real = vid._video_seed_stays_resident
+
+ def _fresh(fam, **kw):
+ kw.pop("device_memory_reader", None)
+ return real(fam, **kw)
+
+ monkeypatch.setattr(vid, "_video_seed_stays_resident", _fresh)
+
+
+def test_auto_seed_settles_once_per_planner_call(monkeypatch):
+ # 16 GiB card with another tenant holding 6 GiB: free is below the settle reserve, so each settle sleeps twice.
+ # An auto fp8 seed first asks whether bf16 stays resident (no), then whether fp8 does: two seed checks.
+ card = _Card(monkeypatch, free_mib = 10 * MIB_PER_GIB, total_mib = 16 * MIB_PER_GIB)
+ backend, fam = _backend(monkeypatch, scheme = "fp8", resident_mib = 0)
+ _plan(backend, fam, "auto")
+ head = (card.reads, card.sleeps)
+
+ with monkeypatch.context() as m:
+ _main_arm(m)
+ card.reads = card.sleeps = 0
+ _plan(backend, fam, "auto")
+ main = (card.reads, card.sleeps)
+
+ assert main == (6, 4) # two settles, three reads and two 1 s sleeps each
+ assert head == (3, 2) # one settle shared by both checks
+
+
+_FREE_GIB = (2, 6, 10, 14, 20, 30, 40, 78)
+_TOTAL_GIB = (16, 24, 32, 48, 80)
+
+
+ "scheme, requested, resident_mib",
+ list(
+ itertools.product(
+ ("nvfp4", "fp8", "int8"), ("auto", None, "nvfp4", "fp8"), (0, 6_000, 20_000)
+ )
+ ),
+)
+def test_planner_decisions_match_the_two_settle_planner(
+ monkeypatch, scheme, requested, resident_mib
+):
+ """Same free/total reading on every read: head (one settle) and main (two settles) choose alike."""
+ card = _Card(monkeypatch, free_mib = 0, total_mib = 0)
+ backend, fam = _backend(monkeypatch, scheme = scheme, resident_mib = resident_mib)
+ head, main = [], []
+ for total in _TOTAL_GIB:
+ for free in _FREE_GIB:
+ if free > total:
+ continue
+ card.free_mib, card.total_mib = free * MIB_PER_GIB, total * MIB_PER_GIB
+ card.reads = 0
+ head.append((total, free, _plan(backend, fam, requested)))
+ assert card.reads <= 3
+ with monkeypatch.context() as m:
+ _main_arm(m)
+ for total in _TOTAL_GIB:
+ for free in _FREE_GIB:
+ if free > total:
+ continue
+ card.free_mib, card.total_mib = free * MIB_PER_GIB, total * MIB_PER_GIB
+ main.append((total, free, _plan(backend, fam, requested)))
+ assert head == main
+ if requested in ("auto", None) and scheme == "fp8" and resident_mib == 0:
+ outcomes = {d for _t, _f, d in head}
+ assert outcomes == {None, vid.DENOISER_SEED_DECLINED, "fp8"}, outcomes
+
+
+def test_reader_failure_still_answers_none(monkeypatch):
+ """A settle that raises leaves the planner's contract unchanged: an unanswerable probe keeps the dense shards."""
+ backend, fam = _backend(monkeypatch, scheme = "nvfp4", resident_mib = 0)
+
+ def _boom(_target, **_kw):
+ raise RuntimeError("no device")
+
+ monkeypatch.setattr(vid, "settled_snapshot_device_memory", _boom)
+ assert _plan(backend, fam, "auto") is None