Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: settle device memory once per video plan, wake sd.cpp polls on server output (#16797)
Daniel Han416cedd19d73
diff --git a/studio/backend/core/inference/sd_cpp_server.py b/studio/backend/core/inference/sd_cpp_server.pyindex b4b94dbf8e8..b31718fa97b 100644--- a/studio/backend/core/inference/sd_cpp_server.py+++ b/studio/backend/core/inference/sd_cpp_server.py@@ -73,6 +73,47 @@ _TERMINAL_CANCELLED = "cancelled"_CANCEL_GRACE_S = 5.0+# Floor between a log line and the next poll; caps log-driven polling at 50/s.+_LOG_WAKE_FLOOR_S = 0.02+++class _LogPacedWait:+ """Sleeps ``interval`` between polls; a server log line cuts it short and starts polls at ``floor``,+ doubling back to ``interval``. sd-server logs just before a status flip or port bind, not at it.+ HTTP status stays the only source of truth."""++ def __init__(+ self,+ wake: Optional[threading.Event],+ interval: float,+ floor: float = _LOG_WAKE_FLOOR_S,+ ) -> None:+ self._wake = wake+ self._interval = max(0.0, float(interval))+ self._floor = max(0.0, min(float(floor), self._interval))+ self._follow: Optional[float] = None+ if wake is not None:+ wake.clear()++ def wait(self) -> None:+ if self._wake is None or self._floor <= 0.0:+ time.sleep(self._interval)+ return+ if self._follow is None:+ woke = self._wake.wait(self._interval)+ else:+ woke = self._wake.wait(self._follow)+ if woke:+ self._wake.clear()+ time.sleep(self._floor)+ self._wake.clear()+ self._follow = self._floor+ return+ if self._follow is not None:+ nxt = self._follow * 2.0+ self._follow = nxt if nxt < self._interval else None++def _socket_owner_is_always_named() -> bool:"""Windows names every socket's owner: an unnamed one there is a psutil quirk (#15022)."""return sys.platform == "win32"@@ -132,6 +173,8 @@ class SdCppServer:self._abort = threading.Event()# Set during a generation so the stdout drain feeds the step-progress callback.self._step_listener: Optional[Callable[[str], None]] = None+ # Set by the stdout drain on every line (and at EOF) so readiness / job polls run sooner.+ self._log_wake = threading.Event()# trust_env=False: loopback-only client must not route through HTTP(S)_PROXY.self._client = httpx.Client(timeout = 30.0, trust_env = False)self._scratch_dir: Optional[str] = None@@ -290,6 +333,7 @@ class SdCppServer:"""A 200 is a true ready signal: upstream binds the port only after the model has loaded."""deadline = time.monotonic() + timeouturl = f"{self.base_url}{_READY_PATH}"+ pacer = _LogPacedWait(getattr(self, "_log_wake", None), interval)while time.monotonic() < deadline:# stop() sets _abort so the wait bails early instead of holding the load for startup_timeout.if self._abort.is_set():@@ -306,7 +350,7 @@ class SdCppServer:return Trueexcept (*_TRANSPORT_ERRORS, httpx.TimeoutException):pass- time.sleep(interval)+ pacer.wait()logger.error("sd-server readiness timed out after %ss", timeout)return False@@ -412,8 +456,16 @@ class SdCppServer:cb(line)except Exception: # noqa: BLE001 -- a progress callback must never break drainpass+ # After the callback, so a woken poll never outruns this line's progress event.+ self._wake_pollers()except Exception: # noqa: BLE001 -- drain thread must never raise (pipe closed at teardown)pass+ self._wake_pollers()++ def _wake_pollers(self) -> None:+ wake = getattr(self, "_log_wake", None)+ if wake is not None:+ wake.set()def reclaimable_params_vram_gb(self, physical_gpu_id: Optional[int]@@ -549,6 +601,8 @@ class SdCppServer:self._step_listener = on_stepjob_id: Optional[str] = None+ # Before submit, so a line logged while the POST is in flight already counts.+ pacer = _LogPacedWait(getattr(self, "_log_wake", None), poll_interval)try:try:resp = self._client.post(@@ -642,7 +696,7 @@ class SdCppServer:raise RuntimeError(message)if status == _TERMINAL_CANCELLED:raise SdCppCancelled("sd-server generation was cancelled.")- time.sleep(poll_interval)+ pacer.wait()finally:self._step_listener = Nonediff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 1d6be78e675..2e78f733486 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -3928,11 +3928,13 @@ def _video_seed_stays_resident(reclaimable_mib: int = 0,hf_token: Optional[str] = None,ltx25_dev: bool = False,+ device_memory_reader: Optional[Callable[[], Any]] = None,) -> bool:"""True when an artifact-sized plan keeps the denoiser resident (text encoders may stream).``scheme=None`` sizes the released bf16 DiT instead, and then also requires a measured budget (see- ``_auto_keeps_resident_bf16``)."""+ ``_auto_keeps_resident_bf16``). ``device_memory_reader`` lets one planner call settle the card once+ for both checks; it must return the raw settled snapshot (reclaimable is applied here)."""from .diffusion_component_sizes import video_components_gbcomponents = video_components_gb(fam, base_repo, allow_hub = True, token = hf_token, dev = ltx25_dev)@@ -3974,7 +3976,11 @@ def _video_seed_stays_resident(from .video_ltx25 import LTX25_WORKER_RESIDENT_GBcompanions_gb += LTX25_WORKER_RESIDENT_GBmodel_dense_mib = int((denoiser_gb + companions_gb) * mib_per_gb)- device_memory = settled_snapshot_device_memory(target)+ device_memory = (+ settled_snapshot_device_memory(target)+ if device_memory_reader is None+ else device_memory_reader()+ )if reclaimable_mib > 0 and device_memory.free_mib is not None:free = device_memory.free_mib + int(reclaimable_mib)if device_memory.total_mib is not None:@@ -7061,6 +7067,14 @@ class VideoBackend:getattr(resident, "pipe", None),ordinal = _target_ordinal(target),)+ settled: list[Any] = []++ def _settled_once() -> Any:+ # Both checks plan on one settled read: each settle can sleep up to 2 s.+ if not settled:+ settled.append(settled_snapshot_device_memory(target))+ return settled[0]+if _auto_seed_yields_to_resident_bf16(scheme, transformer_quant, fam) and _video_seed_stays_resident(@@ -7073,6 +7087,7 @@ class VideoBackend:reclaimable_mib = reclaimable,hf_token = hf_token,ltx25_dev = ltx25_dev,+ device_memory_reader = _settled_once,):# auto keeps a resident bf16 DiT, so the dense shards are what this load runs.return None@@ -7083,12 +7098,10 @@ class VideoBackend:memory_mode = memory_mode,text_encoder_quant = text_encoder_quant,base_repo = base,- reclaimable_mib = _pipeline_device_mib(- getattr(resident, "pipe", None),- ordinal = _target_ordinal(target),- ),+ reclaimable_mib = reclaimable,hf_token = hf_token,ltx25_dev = ltx25_dev,+ device_memory_reader = _settled_once,):logger.info("video.denoiser_prequant: an artifact-sized plan for %s still offloads on "diff --git a/studio/backend/tests/test_sd_cpp_server_log_wake.py b/studio/backend/tests/test_sd_cpp_server_log_wake.pynew file mode 100644index 00000000000..4d0801ef7cb--- /dev/null+++ b/studio/backend/tests/test_sd_cpp_server_log_wake.py@@ -0,0 +1,251 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""sd-server polls wake on server log lines: a real loopback HTTP fake plus a piped stdout.++Upstream sd-server (examples/server/async_jobs.cpp) logs ``generate_image completed`` and then encodes+the PNG before the job flips to ``completed``; ``listening on`` is logged just before the port binds.+Neither flip has its own line, so the poll must follow a line up, not trust it.+"""++from __future__ import annotations++import base64+import io+import json+import os+import socket+import statistics+import threading+import time+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer++import pytest++from core.inference import sd_cpp_server as srv+from core.inference.sd_cpp_server import SdCppServer++_PNG = base64.b64decode(+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="+)+++class _PipeProc:+ """A live child whose stdout is a real pipe the test writes server log lines into."""++ def __init__(self):+ r, w = os.pipe()+ self.stdout = io.TextIOWrapper(io.FileIO(r, "r"), encoding = "utf-8", errors = "replace")+ self._w = io.FileIO(w, "w")+ self.pid = 2**31 - 1 # never a live process+ self.returncode = None++ def log(self, line: str) -> None:+ self._w.write((line + "\n").encode())++ def poll(self):+ return self.returncode++ def terminate(self):+ self.returncode = 0+ try:+ self._w.close()+ except OSError:+ pass++ def kill(self):+ self.terminate()++ def wait(self, timeout = None):+ return self.returncode+++class _FakeSdServer:+ """Loopback job API: one img_gen job that flips to completed ``gap`` after its last log line."""++ def __init__(+ self,+ proc: _PipeProc,+ *,+ step_lines = 3,+ step_s = 0.07,+ gap_s = 0.03,+ ):+ self.proc = proc+ self.step_lines, self.step_s, self.gap_s = step_lines, step_s, gap_s+ self.requests: list[tuple[str, str, bytes]] = []+ self.status = "queued"+ self.flipped_at: float | None = None+ fake = self++ class _H(BaseHTTPRequestHandler):+ def log_message(self, *a):+ pass++ def _send(self, code, payload):+ body = json.dumps(payload).encode()+ self.send_response(code)+ self.send_header("Content-Type", "application/json")+ self.send_header("Content-Length", str(len(body)))+ self.end_headers()+ self.wfile.write(body)++ def do_POST(self):+ body = self.rfile.read(int(self.headers.get("Content-Length") or 0))+ fake.requests.append(("POST", self.path, body))+ threading.Thread(target = fake._run_job, daemon = True).start()+ self._send(202, {"id": "job1", "status": "queued"})++ def do_GET(self):+ fake.requests.append(("GET", self.path, b""))+ if self.path == "/v1/models":+ return self._send(200, {"data": []})+ if fake.status != "completed":+ return self._send(200, {"id": "job1", "status": fake.status, "result": None})+ b64 = base64.b64encode(_PNG).decode()+ self._send(+ 200,+ {+ "id": "job1",+ "status": "completed",+ "result": {"images": [{"index": 0, "b64_json": b64}]},+ },+ )++ self._handler = _H+ self.httpd: ThreadingHTTPServer | None = None++ def bind(self, port: int) -> None:+ self.httpd = ThreadingHTTPServer(("127.0.0.1", port), self._handler)+ threading.Thread(target = self.httpd.serve_forever, daemon = True).start()++ def _run_job(self):+ self.status = "generating"+ for i in range(self.step_lines):+ time.sleep(self.step_s)+ self.proc.log(f"|=====| {i + 1}/{self.step_lines} - 14.2it/s")+ self.proc.log("generate_image completed in 0.25s")+ time.sleep(self.gap_s) # PNG encode + base64 before the status flip, with no line+ self.flipped_at = time.monotonic()+ self.status = "completed"++ def close(self):+ if self.httpd is not None:+ self.httpd.shutdown()+ self.httpd.server_close()+++def _free_port() -> int:+ with socket.socket() as s:+ s.bind(("127.0.0.1", 0))+ return s.getsockname()[1]+++def _attach(proc: _PipeProc, port: int) -> SdCppServer:+ s = SdCppServer("sd-server")+ s._process = proc+ s.port = port+ s._stdout_thread = threading.Thread(target = s._drain_stdout, args = (proc,), daemon = True)+ s._stdout_thread.start()+ return s+++def _one_job(on_step):+ proc = _PipeProc()+ fake = _FakeSdServer(proc)+ port = _free_port()+ fake.bind(port)+ s = _attach(proc, port)+ try:+ images = s.img_gen({"prompt": "a cat", "seed": 7}, on_step = on_step)+ done = time.monotonic()+ return done - fake.flipped_at, images, fake.requests+ finally:+ s.stop()+ fake.close()+++def test_job_completion_is_seen_soon_after_the_last_log_line():+ lags, steps = [], []+ for _ in range(5):+ lag, images, requests = _one_job(steps.append)+ lags.append(lag)+ assert images == [_PNG] # same bytes as served+ posts = [r for r in requests if r[0] == "POST"]+ assert [(m, p, json.loads(b)) for m, p, b in posts] == [+ ("POST", "/sdcpp/v1/img_gen", {"prompt": "a cat", "seed": 7})+ ]+ assert all(p == "/sdcpp/v1/jobs/job1" for m, p, _ in requests if m == "GET")+ # Log-driven polling stays bounded: at most one GET per floor over the job.+ assert len(requests) < 60+ # Every server line reached the progress callback, in order, before the result.+ assert steps == [+ line+ for _ in range(5)+ for line in (+ "|=====| 1/3 - 14.2it/s",+ "|=====| 2/3 - 14.2it/s",+ "|=====| 3/3 - 14.2it/s",+ "generate_image completed in 0.25s",+ )+ ]+ median = statistics.median(lags)+ print(+ f"job flip -> img_gen return lag: median {median * 1000:.1f} ms, all {[round(x * 1000) for x in lags]}"+ )+ assert median < 0.1, lags+++def test_ready_is_seen_soon_after_listening_line(monkeypatch):+ monkeypatch.setattr(SdCppServer, "_port_is_ours", lambda self: True)+ lags = []+ for _ in range(5):+ proc = _PipeProc()+ fake = _FakeSdServer(proc)+ port = _free_port()+ s = _attach(proc, port)+ bound: list[float] = []++ def _serve():+ time.sleep(0.12) # model load+ proc.log(f"listening on: http://127.0.0.1:{port}")+ time.sleep(0.005)+ # Before bind: a ready poll can return before this thread runs again.+ bound.append(time.monotonic())+ fake.bind(port)++ threading.Thread(target = _serve, daemon = True).start()+ try:+ assert s._wait_ready(10.0)+ lags.append(time.monotonic() - bound[0])+ finally:+ s.stop()+ fake.close()+ median = statistics.median(lags)+ print(+ f"bind -> ready lag: median {median * 1000:.1f} ms, all {[round(x * 1000) for x in lags]}"+ )+ # The line comes just before the bind, so the first probe can hit a closed port; Windows takes+ # a few hundred ms to refuse a loopback connect, so there it only has to beat the old 0.5 s poll.+ assert median < (0.5 if os.name == "nt" else 0.1), lags+++def test_timer_still_drives_polls_without_log_lines():+ """No line at all: the fallback timer alone finishes the job (pre-change behaviour)."""+ wait = srv._LogPacedWait(threading.Event(), 0.05)+ t0 = time.monotonic()+ wait.wait()+ assert 0.04 <= time.monotonic() - t0 < 0.5+++def test_follow_up_backs_off_to_the_timer():+ wake = threading.Event()+ wait = srv._LogPacedWait(wake, 0.4, floor = 0.02)+ wake.set()+ wait.wait() # line: one floor+ follows = []+ for _ in range(6):+ follows.append(wait._follow)+ wait.wait()+ assert follows[:4] == [0.02, 0.04, 0.08, 0.16]+ assert follows[5] is None # back on the plain interval timerdiff --git a/studio/backend/tests/test_video_seed_settle_once.py b/studio/backend/tests/test_video_seed_settle_once.pynew file mode 100644index 00000000000..69bd643da7f--- /dev/null+++ b/studio/backend/tests/test_video_seed_settle_once.py@@ -0,0 +1,151 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""The pre-download auto denoiser planner settles the card once for both of its seed checks."""++import itertools+import time+import types++import pytest++torch = pytest.importorskip("torch")++import core.inference.diffusion_component_sizes as dcs+import core.inference.diffusion_memory as dm+import core.inference.video as vid+from core.inference.diffusion_memory import DeviceMemory+from core.inference.video_families import detect_video_family++MIB_PER_GIB = 1024+WAN5B = "Wan-AI/Wan2.2-TI2V-5B-Diffusers"+++class _Card:+ """Fake device-memory provider under the real settle loop: counts reads and settle sleeps."""++ def __init__(self, monkeypatch, free_mib, total_mib):+ self.reads = 0+ self.sleeps = 0+ self.free_mib = free_mib+ self.total_mib = total_mib+ monkeypatch.setattr(dm, "snapshot_device_memory", self._read)+ monkeypatch.setattr(time, "sleep", self._sleep)++ def _read(self, _target):+ self.reads += 1+ return DeviceMemory("cuda", "cuda", "discrete_vram", self.free_mib, self.total_mib)++ def _sleep(self, _s):+ self.sleeps += 1+++def _backend(monkeypatch, *, scheme, resident_mib):+ fam = detect_video_family(WAN5B)+ backend = vid.VideoBackend()+ resident_pipe = object() if resident_mib else None+ backend._state = types.SimpleNamespace(pipe = resident_pipe)+ monkeypatch.setattr(vid, "_video_auto_denoiser_scheme", lambda fam, **kw: scheme)+ monkeypatch.setattr(+ vid,+ "_pipeline_device_mib",+ lambda pipe, ordinal = None: resident_mib if pipe is resident_pipe and pipe else 0,+ )+ monkeypatch.setattr(+ vid,+ "resolve_diffusion_device_target",+ lambda **_k: types.SimpleNamespace(+ device = "cuda", dtype = torch.bfloat16, ordinal = None, supports_model_cpu_offload = True+ ),+ )+ monkeypatch.setattr(+ dcs, "video_components_gb", lambda fam, base, **kw: tuple(fam.bf16_components_gb)+ )+ return backend, fam+++def _plan(backend, fam, requested):+ return backend._video_planned_auto_denoiser_scheme(+ fam, base = None, kind = "pipeline", transformer_quant = requested, speed_mode = None+ )+++def _main_arm(monkeypatch):+ """The pre-fix planner: every seed check runs its own settle."""+ real = vid._video_seed_stays_resident++ def _fresh(fam, **kw):+ kw.pop("device_memory_reader", None)+ return real(fam, **kw)++ monkeypatch.setattr(vid, "_video_seed_stays_resident", _fresh)+++def test_auto_seed_settles_once_per_planner_call(monkeypatch):+ # 16 GiB card with another tenant holding 6 GiB: free is below the settle reserve, so each settle sleeps twice.+ # An auto fp8 seed first asks whether bf16 stays resident (no), then whether fp8 does: two seed checks.+ card = _Card(monkeypatch, free_mib = 10 * MIB_PER_GIB, total_mib = 16 * MIB_PER_GIB)+ backend, fam = _backend(monkeypatch, scheme = "fp8", resident_mib = 0)+ _plan(backend, fam, "auto")+ head = (card.reads, card.sleeps)++ with monkeypatch.context() as m:+ _main_arm(m)+ card.reads = card.sleeps = 0+ _plan(backend, fam, "auto")+ main = (card.reads, card.sleeps)++ assert main == (6, 4) # two settles, three reads and two 1 s sleeps each+ assert head == (3, 2) # one settle shared by both checks+++_FREE_GIB = (2, 6, 10, 14, 20, 30, 40, 78)+_TOTAL_GIB = (16, 24, 32, 48, 80)+++ "scheme, requested, resident_mib",+ list(+ itertools.product(+ ("nvfp4", "fp8", "int8"), ("auto", None, "nvfp4", "fp8"), (0, 6_000, 20_000)+ )+ ),+)+def test_planner_decisions_match_the_two_settle_planner(+ monkeypatch, scheme, requested, resident_mib+):+ """Same free/total reading on every read: head (one settle) and main (two settles) choose alike."""+ card = _Card(monkeypatch, free_mib = 0, total_mib = 0)+ backend, fam = _backend(monkeypatch, scheme = scheme, resident_mib = resident_mib)+ head, main = [], []+ for total in _TOTAL_GIB:+ for free in _FREE_GIB:+ if free > total:+ continue+ card.free_mib, card.total_mib = free * MIB_PER_GIB, total * MIB_PER_GIB+ card.reads = 0+ head.append((total, free, _plan(backend, fam, requested)))+ assert card.reads <= 3+ with monkeypatch.context() as m:+ _main_arm(m)+ for total in _TOTAL_GIB:+ for free in _FREE_GIB:+ if free > total:+ continue+ card.free_mib, card.total_mib = free * MIB_PER_GIB, total * MIB_PER_GIB+ main.append((total, free, _plan(backend, fam, requested)))+ assert head == main+ if requested in ("auto", None) and scheme == "fp8" and resident_mib == 0:+ outcomes = {d for _t, _f, d in head}+ assert outcomes == {None, vid.DENOISER_SEED_DECLINED, "fp8"}, outcomes+++def test_reader_failure_still_answers_none(monkeypatch):+ """A settle that raises leaves the planner's contract unchanged: an unanswerable probe keeps the dense shards."""+ backend, fam = _backend(monkeypatch, scheme = "nvfp4", resident_mib = 0)++ def _boom(_target, **_kw):+ raise RuntimeError("no device")++ monkeypatch.setattr(vid, "settled_snapshot_device_memory", _boom)+ assert _plan(backend, fam, "auto") is None