Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: fail a Deep Research run when no step gathered any evidence (#10663)
Nilay7ead40fce6ad
diff --git a/studio/backend/core/research_runs.py b/studio/backend/core/research_runs.pyindex 73801ff2b58..121e93abc8b 100644--- a/studio/backend/core/research_runs.py+++ b/studio/backend/core/research_runs.py@@ -984,6 +984,19 @@ def _research_step_failed(web_result: str, rag_sources: list[dict]) -> bool:return is_tool_error(web_result) or web_result.strip() in EMPTY_SEARCH_RESULTS+def _preferred_step_error(current: str, candidate: str) -> str:+ """The failure to report when several steps failed differently.++ An engine failure tells the user to wait and retry, an empty sweep tells them to ask+ something else, so a later "No results found." must not bury an earlier rate limit.+ """+ if not candidate:+ return current+ if is_tool_error(current) and not is_tool_error(candidate):+ return current+ return candidate++def _run_moved_on(fresh: dict | None, attempt: int) -> bool:"""Whether the run this worker was running has since been re-pointed at a newer question.@@ -2165,6 +2178,8 @@ class ResearchSupervisor:document_sources: list[dict] = []used_queries: set[str] = set()fetched_urls: set[str] = set()+ completed_steps = 0+ step_error = ""question, conversation_context = await asyncio.to_thread(_research_question_context,run["threadId"],@@ -2191,6 +2206,7 @@ class ResearchSupervisor:elif argument:used_queries.add(argument)if step.get("status") != "completed":+ step_error = _preferred_step_error(step_error, str(result.get("error") or ""))continuerestored_state = _normalize_research_state(result.get("researchState"))if restored_state:@@ -2243,6 +2259,9 @@ class ResearchSupervisor:f"{item.get('text') or item.get('snippet') or ''}"for item in accepted_rag_sources)+ # An unscraped search persists no excerpt, so a completed step can come back with nothing in it.+ if web_evidence or rag_evidence:+ completed_steps += 1title = str(step.get("title") or "Recovered research step")notes.append(f"### {title} ({action})\nInput: {argument}\nResult:\n{web_evidence}\n\n"@@ -2553,6 +2572,10 @@ class ResearchSupervisor:f"Input: {argument}\nResult:\n{result[:12000]}")clean_result = strip_result_for_model(result, "web_search")+ if step_failed:+ step_error = _preferred_step_error(step_error, clean_result[:500])+ else:+ completed_steps += 1step_result = {"action": action["action"],"input": argument,@@ -2597,6 +2620,8 @@ class ResearchSupervisor:)await self._check_worker_write(run["id"], seq is not None)await self._check_active(run["id"])+ if not completed_steps and not sources and not document_sources:+ raise ValueError(f"No research step gathered any evidence. {step_error}".rstrip())source_catalog = "\n".join(f"{index}. Title: {_citation_title(source, source['url'])}\n URL: {source['url']}"for index, source in enumerate(sources, 1)diff --git a/studio/backend/tests/test_research_no_evidence.py b/studio/backend/tests/test_research_no_evidence.pynew file mode 100644index 00000000000..f7163fec6ed--- /dev/null+++ b/studio/backend/tests/test_research_no_evidence.py@@ -0,0 +1,203 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A run whose every step gathered nothing must not be delivered as a finished report."""++from __future__ import annotations++import asyncio+from types import SimpleNamespace++import pytest++from storage import research_runs_db as research_db+from storage import studio_db+++REPORT = "## Findings\n\nWritten from memory, because nothing came back."+THROTTLED = "Search failed: the search engines are rate limiting this machine."+FOUND = (+ "Title: What happened\nURL: https://example.test/what-happened\n"+ "Snippet: It happened on a Tuesday."+)+++def research_home(tmp_path, monkeypatch):+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))+ monkeypatch.setattr(studio_db, "_schema_ready", False)+ studio_db.upsert_chat_thread(+ {+ "id": "thread-1",+ "title": "Research",+ "modelType": "base",+ "modelId": "local-model",+ "createdAt": 1,+ }+ )+ studio_db.upsert_chat_message(+ {+ "id": "user-1",+ "threadId": "thread-1",+ "role": "user",+ "content": [{"type": "text", "text": "what happened?"}],+ "createdAt": 2,+ }+ )+ return tmp_path+++def _claimed_run(+ supervisor, plan_steps: list[dict], max_steps: int, website_policy: dict | None+) -> dict:+ research_db.create_run(+ run_id = "run-1",+ owner_subject = "alice",+ thread_id = "thread-1",+ user_message_id = "user-1",+ assistant_message_id = None,+ config = {+ "model": "local-model",+ "inferenceRequest": {"model": "local-model"},+ "ragScope": None,+ "instructions": "",+ "question": "what happened?",+ "websitePolicy": website_policy,+ "budgets": {+ "maxSteps": max_steps,+ "maxSources": 5,+ "modelTimeoutSeconds": 900,+ "toolTimeoutSeconds": 10,+ },+ },+ )+ planned = research_db.set_plan("run-1", {"title": "Plan", "steps": plan_steps})+ research_db.approve("run-1", planned["planRevision"], planned["planHash"])+ return research_db.claim_next(supervisor.worker_id)+++def _run(+ monkeypatch,+ tool_results: list[str],+ website_policy: dict | None = None,+) -> dict:+ from core import research_runs as worker++ supervisor = worker.ResearchSupervisor(SimpleNamespace(state = SimpleNamespace(server_port = 1)))+ plan_steps = [+ {"title": f"Step {index}", "query": f"what happened {index}"}+ for index in range(len(tool_results))+ ]+ claimed = _claimed_run(supervisor, plan_steps, len(tool_results), website_policy)+ calls = {"n": 0}++ def fake_execute_tool(name, arguments, **kwargs):+ result = tool_results[min(calls["n"], len(tool_results) - 1)]+ calls["n"] += 1+ return result++ async def fake_stream_completion(run, messages, **kwargs):+ if kwargs.get("phase") == "synthesis":+ return REPORT, "", "stop", None+ # Unparseable, so every step falls back to the next unused plan seed.+ return "not json", "", "stop", None++ monkeypatch.setattr(worker, "execute_tool", fake_execute_tool)+ monkeypatch.setattr(supervisor, "_stream_completion", fake_stream_completion)+ asyncio.run(supervisor._process(claimed))+ return research_db.get_run("run-1")+++def test_a_run_whose_every_search_was_throttled_fails_with_the_search_error(+ research_home, monkeypatch+):+ finished = _run(monkeypatch, [THROTTLED, THROTTLED])++ assert finished["status"] == "failed"+ assert "rate limiting this machine" in (finished["error"] or "")+ assert not finished["report"]+ assert not finished["sources"]+++def test_a_run_whose_every_search_matched_nothing_fails(research_home, monkeypatch):+ finished = _run(monkeypatch, ["No results found."])++ assert finished["status"] == "failed"+ assert not finished["report"]+++def test_the_failure_reaches_the_chat_message(research_home, monkeypatch):+ _run(monkeypatch, [THROTTLED])++ messages = studio_db.list_chat_messages("thread-1")+ assistant = [message for message in messages if message["role"] == "assistant"][-1]+ assert assistant["metadata"]["researchStatus"] == "failed"+ assert "rate limiting this machine" in str(assistant["content"])+++def test_a_partially_failed_run_still_completes(research_home, monkeypatch):+ finished = _run(monkeypatch, [FOUND, THROTTLED])++ assert finished["status"] == "completed"+ assert REPORT in finished["report"]+ assert len(finished["sources"]) == 1+++def test_a_step_that_found_results_completes_the_run_even_with_no_citable_source(+ research_home, monkeypatch+):+ """A sweep the website policy filters down to nothing catalogable still gathered the+ evidence in its snippets, and its step is recorded completed."""+ finished = _run(monkeypatch, [FOUND], {"allowedDomains": ["allowed.test"]})++ assert finished["status"] == "completed"+ assert not finished["sources"]+ assert REPORT in finished["report"]+++ ("restored_result", "expected_status"),+ [+ ({"excerpt": "It happened on a Tuesday."}, "completed"),+ ({}, "failed"),+ ],+)+def test_a_resumed_step_counts_only_if_its_evidence_survived(+ research_home, monkeypatch, restored_result, expected_status+):+ from core import research_runs as worker++ supervisor = worker.ResearchSupervisor(SimpleNamespace(state = SimpleNamespace(server_port = 1)))+ plan_steps = [+ {"title": "One", "query": "what happened 0"},+ {"title": "Two", "query": "what happened 1"},+ ]+ _claimed_run(supervisor, plan_steps, 2, None)+ research_db.upsert_execution_step(+ "run-1",+ 0,+ "One",+ "what happened 0",+ "completed",+ {"action": "search", "input": "what happened 0", "sourceCount": 0, **restored_result},+ supervisor.worker_id,+ )+ conn = studio_db.get_connection()+ conn.execute(+ "UPDATE research_runs SET lease_owner = NULL, lease_expires_at = 0 WHERE id = 'run-1'"+ )+ conn.commit()+ conn.close()+ resumed = research_db.claim_next(supervisor.worker_id)+ assert resumed["claimedFromStatus"] == "running"++ async def fake_stream_completion(run, messages, **kwargs):+ if kwargs.get("phase") == "synthesis":+ return REPORT, "", "stop", None+ return "not json", "", "stop", None++ monkeypatch.setattr(worker, "execute_tool", lambda *args, **kwargs: THROTTLED)+ monkeypatch.setattr(supervisor, "_stream_completion", fake_stream_completion)+ asyncio.run(supervisor._process(resumed))++ assert research_db.get_run("run-1")["status"] == expected_statusdiff --git a/studio/backend/tests/test_research_synthesis_recovery.py b/studio/backend/tests/test_research_synthesis_recovery.pyindex 2e06215cbc5..57a06cc9b20 100644--- a/studio/backend/tests/test_research_synthesis_recovery.py+++ b/studio/backend/tests/test_research_synthesis_recovery.py@@ -16,6 +16,12 @@ from storage import studio_dbFIRST_DRAFT = "## Findings\n\n" + ("The evidence says a great deal. " * 120).strip()SHORTER_DRAFT = "## Findings\n\nToo little."+# One gathered source: a run that gathers nothing fails before synthesis, which is not what+# these tests are about.+SEARCH_RESULT = (+ "Title: What happened\nURL: https://example.test/what-happened\n"+ "Snippet: It happened on a Tuesday."+)@pytest.fixture@@ -40,6 +46,9 @@ def research_home(tmp_path, monkeypatch):"createdAt": 2,})+ from core import research_runs as worker++ monkeypatch.setattr(worker, "execute_tool", lambda *args, **kwargs: SEARCH_RESULT)return tmp_path@@ -74,7 +83,9 @@ def _claimed_run(supervisor, external: bool = False) -> dict:},},)- planned = research_db.set_plan("run-1", {"title": "Plan", "steps": []})+ planned = research_db.set_plan(+ "run-1", {"title": "Plan", "steps": [{"title": "Look it up", "query": "what happened"}]}+ )research_db.approve("run-1", planned["planRevision"], planned["planHash"])return research_db.claim_next(supervisor.worker_id)@@ -93,7 +104,7 @@ def _run_synthesis(monkeypatch, *, synthesis, recovery) -> dict:return synthesisif phase == "synthesis_recovery":return recovery- # Unparseable, and an empty plan has no seed action, so the step loop breaks.+ # Unparseable, so the step falls back to the plan's one seed action.return "not json", "", "stop", Nonemonkeypatch.setattr(supervisor, "_stream_completion", fake_stream_completion)