Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: return string error codes from streaming /v1/responses (#10938)
Nilayfeb3335400f7
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 6ad91ed8edd..e26ac38688f 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -29555,6 +29555,12 @@ async def _responses_non_streaming(request_state.skip_api_monitor = previous_skip_monitor+def _responses_error_code(exc: Exception) -> str:+ if _classify_llama_generation_error(exc):+ return "context_length_exceeded"+ return "server_error"++async def _responses_stream(payload: ResponsesRequest,messages: list[ChatMessage],@@ -29659,7 +29665,7 @@ async def _responses_stream(api_monitor.fail(monitor_id, str(exc))raise _openai_admission_http_exception(exc, status_code = 429)- def _responses_admission_failed_sse(exc: Exception, *, status_code: int) -> str:+ def _responses_admission_failed_sse(exc: Exception, *, code: str) -> str:# Emitted in-band after the 200 headers, so flag failed: the middleware must not# treat this failed stream as a success and claim a preview-owned model.mark_response_failed(getattr(request, "scope", None))@@ -29683,7 +29689,7 @@ async def _responses_stream("total_tokens": 0,},"error": {- "code": status_code,+ "code": code,"message": str(exc),},},@@ -30057,7 +30063,7 @@ async def _responses_stream()return [item for _, item in sorted(indexed_items, key = lambda pair: pair[0])]- def _failed_response_payload(exc: Exception, status_code: int) -> dict:+ def _failed_response_payload(exc: Exception) -> dict:# Built only for in-band response.failed events after the 200 headers, so# flag failed: the middleware must not treat a failed stream as a success.mark_response_failed(getattr(request, "scope", None))@@ -30076,7 +30082,7 @@ async def _responses_stream("total_tokens": input_tokens + output_tokens,},"error": {- "code": status_code,+ "code": _responses_error_code(exc),"message": _friendly_error(exc),},},@@ -30145,7 +30151,7 @@ async def _responses_stream("status": "failed","model": _clean_model,"output": [],- "error": {"code": 502, "message": _friendly_error(e)},+ "error": {"code": "server_error", "message": _friendly_error(e)},},},)@@ -30175,7 +30181,7 @@ async def _responses_stream("model": _clean_model,"output": [],"error": {- "code": resp.status_code,+ "code": _responses_error_code(RuntimeError(err_text)),"message": _friendly_upstream_error(err_text[:500]),},},@@ -30219,7 +30225,7 @@ async def _responses_stream(api_monitor.fail(monitor_id, error_message)yield _sse("response.failed",- _failed_response_payload(RuntimeError(error_message), 502),+ _failed_response_payload(RuntimeError(error_message)),)return_apply_usage(chunk_data.get("usage"), chunk_data.get("timings"))@@ -30316,10 +30322,9 @@ async def _responses_stream(if not disconnect_event.is_set():logger.error("responses stream error: %s", e)api_monitor.fail(monitor_id, _friendly_error(e))- status_code = 400 if _classify_llama_generation_error(e) is not None else 500yield _sse("response.failed",- _failed_response_payload(e, status_code),+ _failed_response_payload(e),)returnexcept Exception as e:@@ -30329,10 +30334,9 @@ async def _responses_stream(returnlogger.error("responses stream error: %s", e)api_monitor.fail(monitor_id, _friendly_error(e))- status_code = 400 if _classify_llama_generation_error(e) is not None else 500yield _sse("response.failed",- _failed_response_payload(e, status_code),+ _failed_response_payload(e),)returnfinally:@@ -30665,7 +30669,7 @@ async def _responses_stream(level = "warning",)api_monitor.fail(monitor_id, str(exc))- yield _responses_admission_failed_sse(exc, status_code = 503)+ yield _responses_admission_failed_sse(exc, code = "server_is_overloaded")except LlamaAdmissionCancelled:_llama_admission_log("cancelled-before-upstream",diff --git a/studio/backend/tests/test_responses_tool_passthrough.py b/studio/backend/tests/test_responses_tool_passthrough.pyindex 5d1533de7ce..61a5be2315e 100644--- a/studio/backend/tests/test_responses_tool_passthrough.py+++ b/studio/backend/tests/test_responses_tool_passthrough.py@@ -89,6 +89,11 @@ from fastapi.responses import JSONResponsefrom pydantic import ValidationErrorfrom core.inference.api_monitor import ApiMonitor+from core.inference.llama_admission import (+ ADMISSION_QUEUE_TIMEOUT_ENV,+ LlamaAdmissionConfig,+ get_llama_admission_queue,+)from models.inference import (ChatMessage,ResponsesCustomToolCallInputItem,@@ -3677,3 +3682,117 @@ def test_a_complete_responses_stream_still_ends_on_response_completed(monkeypatcassert completed["response"]["status"] == "completed"assert completed["response"]["incomplete_details"] is Noneassert [item["status"] for item in completed["response"]["output"]] == ["completed"]+++_OVERFLOW_BODY = json.dumps(+ {+ "error": {+ "code": 400,+ "message": "request (16608 tokens) exceeds the available context size (2048 tokens), try increasing it",+ "type": "exceed_context_size_error",+ "n_prompt_tokens": 16608,+ "n_ctx": 2048,+ }+ }+)+++def _failed_stream_error(+ monkeypatch,+ handler = None,+ chunks = (),+):+ import routes.inference as inf_mod++ real_async_client = httpx.AsyncClient+ TestResponsesStreamAdapter._install_stream_mock(monkeypatch, list(chunks))+ if handler is not None:+ monkeypatch.setattr(+ inf_mod.httpx,+ "AsyncClient",+ lambda *args, **kwargs: real_async_client(transport = httpx.MockTransport(handler)),+ )+ payload = ResponsesRequest(input = "hi", stream = True)+ messages = [ChatMessage(role = "user", content = "hi")]++ async def run():+ response = await _responses_stream(payload, messages, TestResponsesStreamAdapter._Request())+ return await TestResponsesStreamAdapter._collect(response)++ [failed] = TestResponsesStreamAdapter._payloads(asyncio.run(run()), "response.failed")+ return failed["response"]["error"]+++ ("status", "body", "code"),+ [+ (400, _OVERFLOW_BODY, "context_length_exceeded"),+ (500, '{"error":{"code":500,"message":"Context size has been exceeded."}}', "server_error"),+ ],+)+def test_upstream_rejection_fails_the_stream_with_a_string_code(monkeypatch, status, body, code):+ error = _failed_stream_error(+ monkeypatch, handler = lambda request: httpx.Response(status, content = body.encode())+ )+ assert error["code"] == code+++ ("message", "code"),+ [+ (+ "request (3868 tokens) exceeds the available context size (2048 tokens), try increasing it",+ "context_length_exceeded",+ ),+ ("Context size has been exceeded.", "server_error"),+ ],+)+def test_in_band_upstream_error_fails_the_stream_with_a_string_code(monkeypatch, message, code):+ error = _failed_stream_error(monkeypatch, chunks = [{"error": {"code": 500, "message": message}}])+ assert error["code"] == code+++def test_unreachable_upstream_fails_the_stream_with_server_error(monkeypatch):+ def refuse(request):+ raise httpx.ConnectError("connection refused", request = request)++ assert _failed_stream_error(monkeypatch, handler = refuse)["code"] == "server_error"+++def test_admission_timeout_fails_the_stream_with_server_is_overloaded(monkeypatch):+ import routes.inference as inf_mod++ async def fail_send(*_args, **_kwargs):+ raise AssertionError("a request that never got a slot must not reach llama-server")++ base_url = "http://llama.responses.admission-timeout.test"+ monkeypatch.setenv(ADMISSION_QUEUE_TIMEOUT_ENV, "0.01")+ monkeypatch.setattr(+ inf_mod,+ "get_llama_cpp_backend",+ lambda: SimpleNamespace(+ is_loaded = True,+ is_vision = False,+ base_url = base_url,+ context_length = 4096,+ effective_parallel_slots = 1,+ _request_reasoning_kwargs = lambda *_args, **_kwargs: None,+ ),+ )+ monkeypatch.setattr(inf_mod, "_send_stream_with_preheader_cancel", fail_send)+ payload = ResponsesRequest(input = "hi", stream = True)+ messages = [ChatMessage(role = "user", content = "hi")]++ async def run():+ queue = get_llama_admission_queue(base_url)+ blocker = queue.reserve(capacity = 1, config = LlamaAdmissionConfig()).lease_nowait()+ try:+ response = await _responses_stream(+ payload, messages, TestResponsesStreamAdapter._Request()+ )+ return await TestResponsesStreamAdapter._collect(response)+ finally:+ blocker.release()++ [failed] = TestResponsesStreamAdapter._payloads(asyncio.run(run()), "response.failed")+ assert failed["response"]["error"]["code"] == "server_is_overloaded"