Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio sd.cpp: image render failures carry sd-server's own output (#16815)
Daniel Han25b53fb91c8a
diff --git a/studio/backend/core/inference/sd_cpp_server.py b/studio/backend/core/inference/sd_cpp_server.pyindex b31718fa97b..a4df444aa16 100644--- a/studio/backend/core/inference/sd_cpp_server.py+++ b/studio/backend/core/inference/sd_cpp_server.py@@ -167,6 +167,7 @@ class SdCppServer:self._resident_params_vram_gb: Optional[float] = None# Bounded tail buffer shared by the drain thread (appends) and readers (diagnostics).self._tail: deque[str] = deque(maxlen = 200)+ self._lines_seen = 0self._stdout_thread: Optional[threading.Thread] = Noneself._lifecycle_lock = threading.Lock()# Set lock-free by stop() so a blocking start()/readiness wait bails promptly.@@ -444,6 +445,7 @@ class SdCppServer:if not line:continueself._tail.append(line)+ self._lines_seen += 1match = _TOTAL_PARAMS_VRAM_RE.search(line)if match is not None:value = float(match.group(1)) / 1024.0@@ -600,6 +602,7 @@ class SdCppServer:raise RuntimeError("sd-server is not running.")self._step_listener = on_step+ job_mark = self._lines_seenjob_id: Optional[str] = None# Before submit, so a line logged while the POST is in flight already counts.pacer = _LogPacedWait(getattr(self, "_log_wake", None), poll_interval)@@ -688,11 +691,13 @@ class SdCppServer:"sd-server generation failed: "f"{err.get('code', 'error')}: {err.get('message', '')}".strip())- if kind == "vid_gen":- # The job error is generic; the cause (OOM, abort) is only in the log.- tail = _diagnostic_tail(self._tail)- if tail:- message += "\nLast output:\n" + tail[:1500]+ # The cause is only in this job's output; an earlier job's stale OOM would mislabel it.+ lines = list(self._tail)+ tail = _diagnostic_tail(+ lines[max(0, len(lines) - (self._lines_seen - job_mark)) :]+ )+ if tail:+ message += "\nLast output:\n" + tail[:1500]raise RuntimeError(message)if status == _TERMINAL_CANCELLED:raise SdCppCancelled("sd-server generation was cancelled.")diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 4b2ec45c578..cb8494c28c7 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -49037,7 +49037,8 @@ _GENERATE_FAILURE_FALLBACK = "Image generation failed."# Failure classes worth naming in the UI, as FIXED text: the engine's own message can embed local paths and argv, so only the class is reported._GENERATE_FAILURE_CLASSES: tuple[tuple[tuple[str, ...], str], ...] = ((- ("out of memory", "outofmemory", "oom"),+ # The last one is stable-diffusion.cpp's model manager: classified here only for a render that failed.+ ("out of memory", "outofmemory", "oom", "cannot make enough memory available"),"The device ran out of memory. Try a smaller size, fewer steps, or a smaller batch.",),(@@ -49062,11 +49063,18 @@ def _generate_failure_detail(message: str) -> str:the engine's text, which can carry local paths and argv."""text = str(message or "").lower()for needles, detail in _GENERATE_FAILURE_CLASSES:- if any(n in text for n in needles):+ if any(_failure_text_mentions(text, n) for n in needles):return f"{_GENERATE_FAILURE_FALLBACK} {detail}"return _GENERATE_FAILURE_FALLBACK+def _failure_text_mentions(text: str, needle: str) -> bool:+ """Bare ``oom`` only as a word: a native renderer's log tail can echo the prompt ("a cozy room")."""+ if needle == "oom":+ return re.search(r"(?<![a-z0-9])oom(?![a-z0-9])", text) is not None+ return needle in text++# Keyed by account id, not account_scope(): the scope moves when the first managed account appears._external_image_runs: dict[str, set] = {}_stopped_external_image_runs: set = set()diff --git a/studio/backend/tests/test_diffusion_routes.py b/studio/backend/tests/test_diffusion_routes.pyindex eda291cea24..732aa4ab97f 100644--- a/studio/backend/tests/test_diffusion_routes.py+++ b/studio/backend/tests/test_diffusion_routes.py@@ -689,6 +689,64 @@ def test_generate_pipeline_error_returns_sanitized_500(client, monkeypatch):assert leak not in detail+def test_generate_native_job_failure_names_an_oom_from_its_log_tail(client, monkeypatch):+ backend = diffusion_module.get_diffusion_backend()+ backend.loaded = True++ def _fail(**kwargs):+ raise RuntimeError(+ "sd-server generation failed: generation_failed: generate_image returned no results\n"+ "Last output:\n[E] ggml: allocating 2304.00 MiB on device 0: cudaMalloc failed: out of memory"+ )++ monkeypatch.setattr(backend, "generate", _fail)+ resp = client.post("/api/inference/images/generate", json = {"prompt": "p"})+ assert resp.status_code == 500+ detail = resp.json()["detail"]+ assert "ran out of memory" in detail+ for leak in ("cudaMalloc", "2304", "generate_image"):+ assert leak not in detail+++def test_generate_native_render_short_of_device_memory_names_the_shortfall(client, monkeypatch):+ backend = diffusion_module.get_diffusion_backend()+ backend.loaded = True++ def _fail(**kwargs):+ raise RuntimeError(+ "sd-server generation failed: generation_failed: generate_image returned no results\n"+ "Last output:\n"+ "[E] MOSS-Image segment 2/43 (moss_image.layers.1) failed during weight preparation\n"+ "[E] sampling for image 1/1 failed after 0.87s\n"+ "[W] model manager cannot make enough memory available on CUDA0: need 1223.37 MB device / 759.59 MB "+ "budget, available 1187.75 MB device / 935.29 MB budget"+ )++ monkeypatch.setattr(backend, "generate", _fail)+ resp = client.post("/api/inference/images/generate", json = {"prompt": "p"})+ assert resp.status_code == 500+ detail = resp.json()["detail"]+ assert "ran out of memory" in detail+ for leak in ("CUDA0", "1223", "segment"):+ assert leak not in detail+++def test_generate_failure_tail_echoing_a_prompt_word_is_not_an_oom(client, monkeypatch):+ backend = diffusion_module.get_diffusion_backend()+ backend.loaded = True++ def _fail(**kwargs):+ raise RuntimeError(+ "sd-server generation failed: generation_failed: generate_image returned no results\n"+ "Last output:\n[I] prompt: a cozy room with a broom, zoomed in"+ )++ monkeypatch.setattr(backend, "generate", _fail)+ resp = client.post("/api/inference/images/generate", json = {"prompt": "p"})+ assert resp.status_code == 500+ assert resp.json()["detail"] == "Image generation failed."++def test_generate_native_process_death_names_the_engine_not_its_output(client, monkeypatch):# Metal hosts: the native renderer aborts inside its text encoderbackend = diffusion_module.get_diffusion_backend()diff --git a/studio/backend/tests/test_h3_native_server.py b/studio/backend/tests/test_h3_native_server.pyindex 488bc6bc1ec..31f7e485ebf 100644--- a/studio/backend/tests/test_h3_native_server.py+++ b/studio/backend/tests/test_h3_native_server.py@@ -147,25 +147,28 @@ def test_vid_gen_other_400_is_a_plain_failure(patched):def test_vid_gen_failure_carries_the_log_cause(patched):- s = _server_with(- _FakePopen(),- _FakeClient(- post = lambda url, json: _Resp(202, {"id": "jobF"}),- get = lambda url: _Resp(- 200,- {- "status": "failed",- "error": {- "code": "generation_failed",- "message": "generate_video returned no results",- },+ client = _FakeClient(+ get = lambda url: _Resp(+ 200,+ {+ "status": "failed",+ "error": {+ "code": "generation_failed",+ "message": "generate_video returned no results",},- ),+ },),)- s._tail.append(- "ggml_backend_cuda_buffer_type_alloc_buffer: allocating 5463.50 MiB: out of memory"- )+ s = _server_with(_FakePopen(), client)++ def post(url, json):+ s._tail.append(+ "ggml_backend_cuda_buffer_type_alloc_buffer: allocating 5463.50 MiB: out of memory"+ )+ s._lines_seen += 1+ return _Resp(202, {"id": "jobF"})++ client._post = postwith pytest.raises(RuntimeError, match = "out of memory"):s.vid_gen({"prompt": "x"}, poll_interval = 0.0)diff --git a/studio/backend/tests/test_sd_cpp_server.py b/studio/backend/tests/test_sd_cpp_server.pyindex f6c1d5dc8f9..01a0542e657 100644--- a/studio/backend/tests/test_sd_cpp_server.py+++ b/studio/backend/tests/test_sd_cpp_server.py@@ -231,6 +231,73 @@ def test_img_gen_failed_job_raises(patched):s.img_gen({"prompt": "x"})+def _emit(server, line):+ server._tail.append(line)+ server._lines_seen += 1+++def _failing_job(server, lines):+ def post(url, json):+ for line in lines:+ _emit(server, line)+ return _Resp(202, {"id": "jobF"})++ return post+++_JOB_FAILED = {+ "status": "failed",+ "error": {"code": "generation_failed", "message": "generate_image returned no results"},+}+_CUDA_OOM = (+ "[E] ggml: ggml_backend_cuda_buffer_type_alloc_buffer: allocating 2304.00 MiB on device 0: "+ "cudaMalloc failed: out of memory"+)+++def test_img_gen_failure_carries_the_log_cause(patched):+ client = _FakeClient(post = None, get = lambda url: _Resp(200, _JOB_FAILED))+ s = _server_with(_FakePopen(), client)+ client._post = _failing_job(s, ["[I] generating image: 1/1 - seed 42", _CUDA_OOM])+ with pytest.raises(RuntimeError) as info:+ s.img_gen({"prompt": "x"}, poll_interval = 0.0)+ message = str(info.value)+ assert message.startswith(+ "sd-server generation failed: generation_failed: generate_image returned no results"+ )+ assert "Last output:" in message+ assert "out of memory" in message+++def test_img_gen_failure_carries_only_its_own_output(patched):+ client = _FakeClient(post = None, get = lambda url: _Resp(200, _JOB_FAILED))+ s = _server_with(_FakePopen(), client)+ client._post = _failing_job(s, [_CUDA_OOM])+ with pytest.raises(RuntimeError):+ s.img_gen({"prompt": "x"}, poll_interval = 0.0)+ client._post = _failing_job(s, ["[E] failed to encode prompt --- image.cpp:450"])+ with pytest.raises(RuntimeError) as info:+ s.img_gen({"prompt": "x"}, poll_interval = 0.0)+ message = str(info.value)+ assert "failed to encode prompt" in message+ assert "out of memory" not in message+++def test_img_gen_failure_without_output_keeps_the_job_error(patched):+ s = _server_with(+ _FakePopen(),+ _FakeClient(+ post = lambda url, json: _Resp(202, {"id": "jobF"}),+ get = lambda url: _Resp(+ 200, {"status": "failed", "error": {"code": "x", "message": "boom"}}+ ),+ ),+ )+ with pytest.raises(RuntimeError) as info:+ s.img_gen({"prompt": "x"}, poll_interval = 0.0)+ assert str(info.value) == "sd-server generation failed: x: boom"++def test_img_gen_queue_full_raises(patched):popen = _FakePopen()s = _server_with(popen, _FakeClient(post = lambda url, json: _Resp(429, text = "busy")))