Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio sd.cpp: image render failures carry sd-server's own output (#16815)

Daniel Han25b53fb91c8a
diff --git a/studio/backend/core/inference/sd_cpp_server.py b/studio/backend/core/inference/sd_cpp_server.py
index b31718fa97b..a4df444aa16 100644
--- a/studio/backend/core/inference/sd_cpp_server.py
+++ b/studio/backend/core/inference/sd_cpp_server.py
@@ -167,6 +167,7 @@ class SdCppServer:
self._resident_params_vram_gb: Optional[float] = None
# Bounded tail buffer shared by the drain thread (appends) and readers (diagnostics).
self._tail: deque[str] = deque(maxlen = 200)
+ self._lines_seen = 0
self._stdout_thread: Optional[threading.Thread] = None
self._lifecycle_lock = threading.Lock()
# Set lock-free by stop() so a blocking start()/readiness wait bails promptly.
@@ -444,6 +445,7 @@ class SdCppServer:
if not line:
continue
self._tail.append(line)
+ self._lines_seen += 1
match = _TOTAL_PARAMS_VRAM_RE.search(line)
if match is not None:
value = float(match.group(1)) / 1024.0
@@ -600,6 +602,7 @@ class SdCppServer:
raise RuntimeError("sd-server is not running.")
self._step_listener = on_step
+ job_mark = self._lines_seen
job_id: Optional[str] = None
# Before submit, so a line logged while the POST is in flight already counts.
pacer = _LogPacedWait(getattr(self, "_log_wake", None), poll_interval)
@@ -688,11 +691,13 @@ class SdCppServer:
"sd-server generation failed: "
f"{err.get('code', 'error')}: {err.get('message', '')}".strip()
)
- if kind == "vid_gen":
- # The job error is generic; the cause (OOM, abort) is only in the log.
- tail = _diagnostic_tail(self._tail)
- if tail:
- message += "\nLast output:\n" + tail[:1500]
+ # The cause is only in this job's output; an earlier job's stale OOM would mislabel it.
+ lines = list(self._tail)
+ tail = _diagnostic_tail(
+ lines[max(0, len(lines) - (self._lines_seen - job_mark)) :]
+ )
+ if tail:
+ message += "\nLast output:\n" + tail[:1500]
raise RuntimeError(message)
if status == _TERMINAL_CANCELLED:
raise SdCppCancelled("sd-server generation was cancelled.")
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 4b2ec45c578..cb8494c28c7 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -49037,7 +49037,8 @@ _GENERATE_FAILURE_FALLBACK = "Image generation failed."
# Failure classes worth naming in the UI, as FIXED text: the engine's own message can embed local paths and argv, so only the class is reported.
_GENERATE_FAILURE_CLASSES: tuple[tuple[tuple[str, ...], str], ...] = (
(
- ("out of memory", "outofmemory", "oom"),
+ # The last one is stable-diffusion.cpp's model manager: classified here only for a render that failed.
+ ("out of memory", "outofmemory", "oom", "cannot make enough memory available"),
"The device ran out of memory. Try a smaller size, fewer steps, or a smaller batch.",
),
(
@@ -49062,11 +49063,18 @@ def _generate_failure_detail(message: str) -> str:
the engine's text, which can carry local paths and argv."""
text = str(message or "").lower()
for needles, detail in _GENERATE_FAILURE_CLASSES:
- if any(n in text for n in needles):
+ if any(_failure_text_mentions(text, n) for n in needles):
return f"{_GENERATE_FAILURE_FALLBACK} {detail}"
return _GENERATE_FAILURE_FALLBACK
+def _failure_text_mentions(text: str, needle: str) -> bool:
+ """Bare ``oom`` only as a word: a native renderer's log tail can echo the prompt ("a cozy room")."""
+ if needle == "oom":
+ return re.search(r"(?<![a-z0-9])oom(?![a-z0-9])", text) is not None
+ return needle in text
+
+
# Keyed by account id, not account_scope(): the scope moves when the first managed account appears.
_external_image_runs: dict[str, set] = {}
_stopped_external_image_runs: set = set()
diff --git a/studio/backend/tests/test_diffusion_routes.py b/studio/backend/tests/test_diffusion_routes.py
index eda291cea24..732aa4ab97f 100644
--- a/studio/backend/tests/test_diffusion_routes.py
+++ b/studio/backend/tests/test_diffusion_routes.py
@@ -689,6 +689,64 @@ def test_generate_pipeline_error_returns_sanitized_500(client, monkeypatch):
assert leak not in detail
+def test_generate_native_job_failure_names_an_oom_from_its_log_tail(client, monkeypatch):
+ backend = diffusion_module.get_diffusion_backend()
+ backend.loaded = True
+
+ def _fail(**kwargs):
+ raise RuntimeError(
+ "sd-server generation failed: generation_failed: generate_image returned no results\n"
+ "Last output:\n[E] ggml: allocating 2304.00 MiB on device 0: cudaMalloc failed: out of memory"
+ )
+
+ monkeypatch.setattr(backend, "generate", _fail)
+ resp = client.post("/api/inference/images/generate", json = {"prompt": "p"})
+ assert resp.status_code == 500
+ detail = resp.json()["detail"]
+ assert "ran out of memory" in detail
+ for leak in ("cudaMalloc", "2304", "generate_image"):
+ assert leak not in detail
+
+
+def test_generate_native_render_short_of_device_memory_names_the_shortfall(client, monkeypatch):
+ backend = diffusion_module.get_diffusion_backend()
+ backend.loaded = True
+
+ def _fail(**kwargs):
+ raise RuntimeError(
+ "sd-server generation failed: generation_failed: generate_image returned no results\n"
+ "Last output:\n"
+ "[E] MOSS-Image segment 2/43 (moss_image.layers.1) failed during weight preparation\n"
+ "[E] sampling for image 1/1 failed after 0.87s\n"
+ "[W] model manager cannot make enough memory available on CUDA0: need 1223.37 MB device / 759.59 MB "
+ "budget, available 1187.75 MB device / 935.29 MB budget"
+ )
+
+ monkeypatch.setattr(backend, "generate", _fail)
+ resp = client.post("/api/inference/images/generate", json = {"prompt": "p"})
+ assert resp.status_code == 500
+ detail = resp.json()["detail"]
+ assert "ran out of memory" in detail
+ for leak in ("CUDA0", "1223", "segment"):
+ assert leak not in detail
+
+
+def test_generate_failure_tail_echoing_a_prompt_word_is_not_an_oom(client, monkeypatch):
+ backend = diffusion_module.get_diffusion_backend()
+ backend.loaded = True
+
+ def _fail(**kwargs):
+ raise RuntimeError(
+ "sd-server generation failed: generation_failed: generate_image returned no results\n"
+ "Last output:\n[I] prompt: a cozy room with a broom, zoomed in"
+ )
+
+ monkeypatch.setattr(backend, "generate", _fail)
+ resp = client.post("/api/inference/images/generate", json = {"prompt": "p"})
+ assert resp.status_code == 500
+ assert resp.json()["detail"] == "Image generation failed."
+
+
def test_generate_native_process_death_names_the_engine_not_its_output(client, monkeypatch):
# Metal hosts: the native renderer aborts inside its text encoder
backend = diffusion_module.get_diffusion_backend()
diff --git a/studio/backend/tests/test_h3_native_server.py b/studio/backend/tests/test_h3_native_server.py
index 488bc6bc1ec..31f7e485ebf 100644
--- a/studio/backend/tests/test_h3_native_server.py
+++ b/studio/backend/tests/test_h3_native_server.py
@@ -147,25 +147,28 @@ def test_vid_gen_other_400_is_a_plain_failure(patched):
def test_vid_gen_failure_carries_the_log_cause(patched):
- s = _server_with(
- _FakePopen(),
- _FakeClient(
- post = lambda url, json: _Resp(202, {"id": "jobF"}),
- get = lambda url: _Resp(
- 200,
- {
- "status": "failed",
- "error": {
- "code": "generation_failed",
- "message": "generate_video returned no results",
- },
+ client = _FakeClient(
+ get = lambda url: _Resp(
+ 200,
+ {
+ "status": "failed",
+ "error": {
+ "code": "generation_failed",
+ "message": "generate_video returned no results",
},
- ),
+ },
),
)
- s._tail.append(
- "ggml_backend_cuda_buffer_type_alloc_buffer: allocating 5463.50 MiB: out of memory"
- )
+ s = _server_with(_FakePopen(), client)
+
+ def post(url, json):
+ s._tail.append(
+ "ggml_backend_cuda_buffer_type_alloc_buffer: allocating 5463.50 MiB: out of memory"
+ )
+ s._lines_seen += 1
+ return _Resp(202, {"id": "jobF"})
+
+ client._post = post
with pytest.raises(RuntimeError, match = "out of memory"):
s.vid_gen({"prompt": "x"}, poll_interval = 0.0)
diff --git a/studio/backend/tests/test_sd_cpp_server.py b/studio/backend/tests/test_sd_cpp_server.py
index f6c1d5dc8f9..01a0542e657 100644
--- a/studio/backend/tests/test_sd_cpp_server.py
+++ b/studio/backend/tests/test_sd_cpp_server.py
@@ -231,6 +231,73 @@ def test_img_gen_failed_job_raises(patched):
s.img_gen({"prompt": "x"})
+def _emit(server, line):
+ server._tail.append(line)
+ server._lines_seen += 1
+
+
+def _failing_job(server, lines):
+ def post(url, json):
+ for line in lines:
+ _emit(server, line)
+ return _Resp(202, {"id": "jobF"})
+
+ return post
+
+
+_JOB_FAILED = {
+ "status": "failed",
+ "error": {"code": "generation_failed", "message": "generate_image returned no results"},
+}
+_CUDA_OOM = (
+ "[E] ggml: ggml_backend_cuda_buffer_type_alloc_buffer: allocating 2304.00 MiB on device 0: "
+ "cudaMalloc failed: out of memory"
+)
+
+
+def test_img_gen_failure_carries_the_log_cause(patched):
+ client = _FakeClient(post = None, get = lambda url: _Resp(200, _JOB_FAILED))
+ s = _server_with(_FakePopen(), client)
+ client._post = _failing_job(s, ["[I] generating image: 1/1 - seed 42", _CUDA_OOM])
+ with pytest.raises(RuntimeError) as info:
+ s.img_gen({"prompt": "x"}, poll_interval = 0.0)
+ message = str(info.value)
+ assert message.startswith(
+ "sd-server generation failed: generation_failed: generate_image returned no results"
+ )
+ assert "Last output:" in message
+ assert "out of memory" in message
+
+
+def test_img_gen_failure_carries_only_its_own_output(patched):
+ client = _FakeClient(post = None, get = lambda url: _Resp(200, _JOB_FAILED))
+ s = _server_with(_FakePopen(), client)
+ client._post = _failing_job(s, [_CUDA_OOM])
+ with pytest.raises(RuntimeError):
+ s.img_gen({"prompt": "x"}, poll_interval = 0.0)
+ client._post = _failing_job(s, ["[E] failed to encode prompt --- image.cpp:450"])
+ with pytest.raises(RuntimeError) as info:
+ s.img_gen({"prompt": "x"}, poll_interval = 0.0)
+ message = str(info.value)
+ assert "failed to encode prompt" in message
+ assert "out of memory" not in message
+
+
+def test_img_gen_failure_without_output_keeps_the_job_error(patched):
+ s = _server_with(
+ _FakePopen(),
+ _FakeClient(
+ post = lambda url, json: _Resp(202, {"id": "jobF"}),
+ get = lambda url: _Resp(
+ 200, {"status": "failed", "error": {"code": "x", "message": "boom"}}
+ ),
+ ),
+ )
+ with pytest.raises(RuntimeError) as info:
+ s.img_gen({"prompt": "x"}, poll_interval = 0.0)
+ assert str(info.value) == "sd-server generation failed: x: boom"
+
+
def test_img_gen_queue_full_raises(patched):
popen = _FakePopen()
s = _server_with(popen, _FakeClient(post = lambda url, json: _Resp(429, text = "busy")))