Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: return string error codes from streaming /v1/responses (#10938)

Nilayfeb3335400f7
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 6ad91ed8edd..e26ac38688f 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -29555,6 +29555,12 @@ async def _responses_non_streaming(
request_state.skip_api_monitor = previous_skip_monitor
+def _responses_error_code(exc: Exception) -> str:
+ if _classify_llama_generation_error(exc):
+ return "context_length_exceeded"
+ return "server_error"
+
+
async def _responses_stream(
payload: ResponsesRequest,
messages: list[ChatMessage],
@@ -29659,7 +29665,7 @@ async def _responses_stream(
api_monitor.fail(monitor_id, str(exc))
raise _openai_admission_http_exception(exc, status_code = 429)
- def _responses_admission_failed_sse(exc: Exception, *, status_code: int) -> str:
+ def _responses_admission_failed_sse(exc: Exception, *, code: str) -> str:
# Emitted in-band after the 200 headers, so flag failed: the middleware must not
# treat this failed stream as a success and claim a preview-owned model.
mark_response_failed(getattr(request, "scope", None))
@@ -29683,7 +29689,7 @@ async def _responses_stream(
"total_tokens": 0,
},
"error": {
- "code": status_code,
+ "code": code,
"message": str(exc),
},
},
@@ -30057,7 +30063,7 @@ async def _responses_stream(
)
return [item for _, item in sorted(indexed_items, key = lambda pair: pair[0])]
- def _failed_response_payload(exc: Exception, status_code: int) -> dict:
+ def _failed_response_payload(exc: Exception) -> dict:
# Built only for in-band response.failed events after the 200 headers, so
# flag failed: the middleware must not treat a failed stream as a success.
mark_response_failed(getattr(request, "scope", None))
@@ -30076,7 +30082,7 @@ async def _responses_stream(
"total_tokens": input_tokens + output_tokens,
},
"error": {
- "code": status_code,
+ "code": _responses_error_code(exc),
"message": _friendly_error(exc),
},
},
@@ -30145,7 +30151,7 @@ async def _responses_stream(
"status": "failed",
"model": _clean_model,
"output": [],
- "error": {"code": 502, "message": _friendly_error(e)},
+ "error": {"code": "server_error", "message": _friendly_error(e)},
},
},
)
@@ -30175,7 +30181,7 @@ async def _responses_stream(
"model": _clean_model,
"output": [],
"error": {
- "code": resp.status_code,
+ "code": _responses_error_code(RuntimeError(err_text)),
"message": _friendly_upstream_error(err_text[:500]),
},
},
@@ -30219,7 +30225,7 @@ async def _responses_stream(
api_monitor.fail(monitor_id, error_message)
yield _sse(
"response.failed",
- _failed_response_payload(RuntimeError(error_message), 502),
+ _failed_response_payload(RuntimeError(error_message)),
)
return
_apply_usage(chunk_data.get("usage"), chunk_data.get("timings"))
@@ -30316,10 +30322,9 @@ async def _responses_stream(
if not disconnect_event.is_set():
logger.error("responses stream error: %s", e)
api_monitor.fail(monitor_id, _friendly_error(e))
- status_code = 400 if _classify_llama_generation_error(e) is not None else 500
yield _sse(
"response.failed",
- _failed_response_payload(e, status_code),
+ _failed_response_payload(e),
)
return
except Exception as e:
@@ -30329,10 +30334,9 @@ async def _responses_stream(
return
logger.error("responses stream error: %s", e)
api_monitor.fail(monitor_id, _friendly_error(e))
- status_code = 400 if _classify_llama_generation_error(e) is not None else 500
yield _sse(
"response.failed",
- _failed_response_payload(e, status_code),
+ _failed_response_payload(e),
)
return
finally:
@@ -30665,7 +30669,7 @@ async def _responses_stream(
level = "warning",
)
api_monitor.fail(monitor_id, str(exc))
- yield _responses_admission_failed_sse(exc, status_code = 503)
+ yield _responses_admission_failed_sse(exc, code = "server_is_overloaded")
except LlamaAdmissionCancelled:
_llama_admission_log(
"cancelled-before-upstream",
diff --git a/studio/backend/tests/test_responses_tool_passthrough.py b/studio/backend/tests/test_responses_tool_passthrough.py
index 5d1533de7ce..61a5be2315e 100644
--- a/studio/backend/tests/test_responses_tool_passthrough.py
+++ b/studio/backend/tests/test_responses_tool_passthrough.py
@@ -89,6 +89,11 @@ from fastapi.responses import JSONResponse
from pydantic import ValidationError
from core.inference.api_monitor import ApiMonitor
+from core.inference.llama_admission import (
+ ADMISSION_QUEUE_TIMEOUT_ENV,
+ LlamaAdmissionConfig,
+ get_llama_admission_queue,
+)
from models.inference import (
ChatMessage,
ResponsesCustomToolCallInputItem,
@@ -3677,3 +3682,117 @@ def test_a_complete_responses_stream_still_ends_on_response_completed(monkeypatc
assert completed["response"]["status"] == "completed"
assert completed["response"]["incomplete_details"] is None
assert [item["status"] for item in completed["response"]["output"]] == ["completed"]
+
+
+_OVERFLOW_BODY = json.dumps(
+ {
+ "error": {
+ "code": 400,
+ "message": "request (16608 tokens) exceeds the available context size (2048 tokens), try increasing it",
+ "type": "exceed_context_size_error",
+ "n_prompt_tokens": 16608,
+ "n_ctx": 2048,
+ }
+ }
+)
+
+
+def _failed_stream_error(
+ monkeypatch,
+ handler = None,
+ chunks = (),
+):
+ import routes.inference as inf_mod
+
+ real_async_client = httpx.AsyncClient
+ TestResponsesStreamAdapter._install_stream_mock(monkeypatch, list(chunks))
+ if handler is not None:
+ monkeypatch.setattr(
+ inf_mod.httpx,
+ "AsyncClient",
+ lambda *args, **kwargs: real_async_client(transport = httpx.MockTransport(handler)),
+ )
+ payload = ResponsesRequest(input = "hi", stream = True)
+ messages = [ChatMessage(role = "user", content = "hi")]
+
+ async def run():
+ response = await _responses_stream(payload, messages, TestResponsesStreamAdapter._Request())
+ return await TestResponsesStreamAdapter._collect(response)
+
+ [failed] = TestResponsesStreamAdapter._payloads(asyncio.run(run()), "response.failed")
+ return failed["response"]["error"]
+
+
+ ("status", "body", "code"),
+ [
+ (400, _OVERFLOW_BODY, "context_length_exceeded"),
+ (500, '{"error":{"code":500,"message":"Context size has been exceeded."}}', "server_error"),
+ ],
+)
+def test_upstream_rejection_fails_the_stream_with_a_string_code(monkeypatch, status, body, code):
+ error = _failed_stream_error(
+ monkeypatch, handler = lambda request: httpx.Response(status, content = body.encode())
+ )
+ assert error["code"] == code
+
+
+ ("message", "code"),
+ [
+ (
+ "request (3868 tokens) exceeds the available context size (2048 tokens), try increasing it",
+ "context_length_exceeded",
+ ),
+ ("Context size has been exceeded.", "server_error"),
+ ],
+)
+def test_in_band_upstream_error_fails_the_stream_with_a_string_code(monkeypatch, message, code):
+ error = _failed_stream_error(monkeypatch, chunks = [{"error": {"code": 500, "message": message}}])
+ assert error["code"] == code
+
+
+def test_unreachable_upstream_fails_the_stream_with_server_error(monkeypatch):
+ def refuse(request):
+ raise httpx.ConnectError("connection refused", request = request)
+
+ assert _failed_stream_error(monkeypatch, handler = refuse)["code"] == "server_error"
+
+
+def test_admission_timeout_fails_the_stream_with_server_is_overloaded(monkeypatch):
+ import routes.inference as inf_mod
+
+ async def fail_send(*_args, **_kwargs):
+ raise AssertionError("a request that never got a slot must not reach llama-server")
+
+ base_url = "http://llama.responses.admission-timeout.test"
+ monkeypatch.setenv(ADMISSION_QUEUE_TIMEOUT_ENV, "0.01")
+ monkeypatch.setattr(
+ inf_mod,
+ "get_llama_cpp_backend",
+ lambda: SimpleNamespace(
+ is_loaded = True,
+ is_vision = False,
+ base_url = base_url,
+ context_length = 4096,
+ effective_parallel_slots = 1,
+ _request_reasoning_kwargs = lambda *_args, **_kwargs: None,
+ ),
+ )
+ monkeypatch.setattr(inf_mod, "_send_stream_with_preheader_cancel", fail_send)
+ payload = ResponsesRequest(input = "hi", stream = True)
+ messages = [ChatMessage(role = "user", content = "hi")]
+
+ async def run():
+ queue = get_llama_admission_queue(base_url)
+ blocker = queue.reserve(capacity = 1, config = LlamaAdmissionConfig()).lease_nowait()
+ try:
+ response = await _responses_stream(
+ payload, messages, TestResponsesStreamAdapter._Request()
+ )
+ return await TestResponsesStreamAdapter._collect(response)
+ finally:
+ blocker.release()
+
+ [failed] = TestResponsesStreamAdapter._payloads(asyncio.run(run()), "response.failed")
+ assert failed["response"]["error"]["code"] == "server_is_overloaded"