Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: answer on Mistral reasoning models instead of an internal error (#17542)
Nilay7cda598cd7ec
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 02514522aae..7a9d0553d29 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -3762,6 +3762,7 @@ from core.inference.sse_control_frames import (from core.inference.studio_tool_loop import (ToolLoopPolicy,ToolLoopRun,+ _delta_text,stream_with_studio_tools,)from core.inference.chat_templates import resolve_effective_chat_template_override@@ -7568,7 +7569,19 @@ def _monitor_openai_chunk(choice_index = idxdelta = choice.get("delta") or {}message = choice.get("message") or {}- if isinstance(delta, dict) and delta.get("reasoning_content"):+ raw_content = delta.get("content") if isinstance(delta, dict) else None+ if isinstance(delta, dict) and (+ delta.get("reasoning_content")+ or (+ isinstance(raw_content, list)+ and any(+ isinstance(part, dict)+ and part.get("type") == "thinking"+ and _delta_text(part.get("thinking"))+ for part in raw_content+ )+ )+ ):api_monitor.mark_first_token(monitor_id)streamed_tool_delta = False@@ -7584,7 +7597,7 @@ def _monitor_openai_chunk(choice_index,delta.get("function_call"),)- content = delta.get("content") if isinstance(delta, dict) else None+ content = _delta_text(raw_content)if content:if streaming:_flush_monitor_stream_tool_calls(@@ -7684,7 +7697,10 @@ def _monitor_openai_sse_data(if error_message:api_monitor.fail(monitor_id, error_message)return "error"- _monitor_openai_chunk(monitor_id, data, context_length, streaming = True)+ try:+ _monitor_openai_chunk(monitor_id, data, context_length, streaming = True)+ except Exception as exc:+ logger.warning("api_monitor.chunk_failed", error = str(exc))return Nonediff --git a/studio/backend/tests/test_api_monitor.py b/studio/backend/tests/test_api_monitor.pyindex efffffc7276..50921872c49 100644--- a/studio/backend/tests/test_api_monitor.py+++ b/studio/backend/tests/test_api_monitor.py@@ -1628,3 +1628,108 @@ def test_progress_updates_advance_updated_at_on_a_frozen_clock(monkeypatch):seen.append(monitor.snapshot()[0]["updated_at"])assert seen[0] < seen[1] < seen[2]+++def test_mistral_reasoning_stream_reaches_the_client_and_the_monitor(monkeypatch):+ import asyncio+ from types import SimpleNamespace++ import httpx++ from core.inference import external_provider as ep+ from models.inference import ChatCompletionRequest++ thinking = [{"type": "thinking", "thinking": [{"type": "text", "text": "Multiply 17 by 23."}]}]+ deltas = [+ {"role": "assistant", "content": thinking},+ {+ "content": [+ {"type": "thinking", "thinking": [{"type": "text", "text": ""}]},+ {"type": "text", "text": "17 * 23"},+ ]+ },+ {"content": " = 391"},+ ]+ chunks = [+ {"choices": [{"index": 0, "delta": delta, "finish_reason": None}]} for delta in deltas+ ]+ chunks.append({"choices": [{"index": 0, "delta": {"content": ""}, "finish_reason": "stop"}]})+ sse = "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + "data: [DONE]\n\n"++ def handle(_request):+ return httpx.Response(+ 200, headers = {"content-type": "text/event-stream"}, content = sse.encode()+ )++ async def run():+ async with httpx.AsyncClient(transport = httpx.MockTransport(handle)) as client:+ monkeypatch.setattr(ep, "_http_client", client)+ monitor = ApiMonitor(max_entries = 3)+ monkeypatch.setattr(inference_route, "api_monitor", monitor)+ payload = ChatCompletionRequest(+ messages = [{"role": "user", "content": "What is 17 * 23?"}],+ stream = True,+ provider_type = "mistral",+ external_model = "mistral-medium-latest",+ )++ async def disconnected():+ return False++ request = SimpleNamespace(+ headers = {},+ state = SimpleNamespace(skip_api_monitor = False),+ url = SimpleNamespace(path = "/v1/chat/completions"),+ method = "POST",+ is_disconnected = disconnected,+ )+ response = await inference_route._proxy_to_external_provider(payload, request)+ body = [part async for part in response.body_iterator]+ return body, monitor++ body, monitor = asyncio.run(run())+ sent = []+ for part in body:+ part = part.decode() if isinstance(part, bytes) else part+ for line in part.splitlines():+ if line.startswith("data: {"):+ sent.append(json.loads(line[6:]))+ assert not [chunk for chunk in sent if "error" in chunk]+ contents = [+ choice["delta"]["content"]+ for chunk in sent+ for choice in chunk.get("choices") or []+ if choice.get("delta", {}).get("content")+ ]+ assert contents == [delta["content"] for delta in deltas]+ row = monitor.snapshot()[0]+ assert row["status"] == "completed"+ assert row["reply"] == "17 * 23 = 391"+++def test_a_monitor_failure_does_not_abort_the_stream(monkeypatch):+ monitor = ApiMonitor(max_entries = 3)+ monkeypatch.setattr(inference_route, "api_monitor", monitor)+ entry_id = _start(monitor)++ def boom(*_args, **_kwargs):+ raise RuntimeError("monitor bug")++ monkeypatch.setattr(monitor, "append_reply", boom)+ line = "data: " + json.dumps({"choices": [{"index": 0, "delta": {"content": "hi"}}]})++ assert inference_route._monitor_openai_sse_line(entry_id, line) is None+++def test_mistral_thinking_part_starts_token_generation(monkeypatch):+ monitor = ApiMonitor(max_entries = 3)+ monkeypatch.setattr(inference_route, "api_monitor", monitor)+ entry_id = _start(monitor)+ thinking = [{"type": "thinking", "thinking": [{"type": "text", "text": "Hmm."}]}]+ line = "data: " + json.dumps({"choices": [{"index": 0, "delta": {"content": thinking}}]})++ inference_route._monitor_openai_sse_line(entry_id, line)++ row = monitor.snapshot()[0]+ assert row["running_phase"] == "token_generation"+ assert row["reply"] == ""