Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: fail a Deep Research run when no step gathered any evidence (#10663)

Nilay7ead40fce6ad
diff --git a/studio/backend/core/research_runs.py b/studio/backend/core/research_runs.py
index 73801ff2b58..121e93abc8b 100644
--- a/studio/backend/core/research_runs.py
+++ b/studio/backend/core/research_runs.py
@@ -984,6 +984,19 @@ def _research_step_failed(web_result: str, rag_sources: list[dict]) -> bool:
return is_tool_error(web_result) or web_result.strip() in EMPTY_SEARCH_RESULTS
+def _preferred_step_error(current: str, candidate: str) -> str:
+ """The failure to report when several steps failed differently.
+
+ An engine failure tells the user to wait and retry, an empty sweep tells them to ask
+ something else, so a later "No results found." must not bury an earlier rate limit.
+ """
+ if not candidate:
+ return current
+ if is_tool_error(current) and not is_tool_error(candidate):
+ return current
+ return candidate
+
+
def _run_moved_on(fresh: dict | None, attempt: int) -> bool:
"""Whether the run this worker was running has since been re-pointed at a newer question.
@@ -2165,6 +2178,8 @@ class ResearchSupervisor:
document_sources: list[dict] = []
used_queries: set[str] = set()
fetched_urls: set[str] = set()
+ completed_steps = 0
+ step_error = ""
question, conversation_context = await asyncio.to_thread(
_research_question_context,
run["threadId"],
@@ -2191,6 +2206,7 @@ class ResearchSupervisor:
elif argument:
used_queries.add(argument)
if step.get("status") != "completed":
+ step_error = _preferred_step_error(step_error, str(result.get("error") or ""))
continue
restored_state = _normalize_research_state(result.get("researchState"))
if restored_state:
@@ -2243,6 +2259,9 @@ class ResearchSupervisor:
f"{item.get('text') or item.get('snippet') or ''}"
for item in accepted_rag_sources
)
+ # An unscraped search persists no excerpt, so a completed step can come back with nothing in it.
+ if web_evidence or rag_evidence:
+ completed_steps += 1
title = str(step.get("title") or "Recovered research step")
notes.append(
f"### {title} ({action})\nInput: {argument}\nResult:\n{web_evidence}\n\n"
@@ -2553,6 +2572,10 @@ class ResearchSupervisor:
f"Input: {argument}\nResult:\n{result[:12000]}"
)
clean_result = strip_result_for_model(result, "web_search")
+ if step_failed:
+ step_error = _preferred_step_error(step_error, clean_result[:500])
+ else:
+ completed_steps += 1
step_result = {
"action": action["action"],
"input": argument,
@@ -2597,6 +2620,8 @@ class ResearchSupervisor:
)
await self._check_worker_write(run["id"], seq is not None)
await self._check_active(run["id"])
+ if not completed_steps and not sources and not document_sources:
+ raise ValueError(f"No research step gathered any evidence. {step_error}".rstrip())
source_catalog = "\n".join(
f"{index}. Title: {_citation_title(source, source['url'])}\n URL: {source['url']}"
for index, source in enumerate(sources, 1)
diff --git a/studio/backend/tests/test_research_no_evidence.py b/studio/backend/tests/test_research_no_evidence.py
new file mode 100644
index 00000000000..f7163fec6ed
--- /dev/null
+++ b/studio/backend/tests/test_research_no_evidence.py
@@ -0,0 +1,203 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A run whose every step gathered nothing must not be delivered as a finished report."""
+
+from __future__ import annotations
+
+import asyncio
+from types import SimpleNamespace
+
+import pytest
+
+from storage import research_runs_db as research_db
+from storage import studio_db
+
+
+REPORT = "## Findings\n\nWritten from memory, because nothing came back."
+THROTTLED = "Search failed: the search engines are rate limiting this machine."
+FOUND = (
+ "Title: What happened\nURL: https://example.test/what-happened\n"
+ "Snippet: It happened on a Tuesday."
+)
+
+
+def research_home(tmp_path, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
+ monkeypatch.setattr(studio_db, "_schema_ready", False)
+ studio_db.upsert_chat_thread(
+ {
+ "id": "thread-1",
+ "title": "Research",
+ "modelType": "base",
+ "modelId": "local-model",
+ "createdAt": 1,
+ }
+ )
+ studio_db.upsert_chat_message(
+ {
+ "id": "user-1",
+ "threadId": "thread-1",
+ "role": "user",
+ "content": [{"type": "text", "text": "what happened?"}],
+ "createdAt": 2,
+ }
+ )
+ return tmp_path
+
+
+def _claimed_run(
+ supervisor, plan_steps: list[dict], max_steps: int, website_policy: dict | None
+) -> dict:
+ research_db.create_run(
+ run_id = "run-1",
+ owner_subject = "alice",
+ thread_id = "thread-1",
+ user_message_id = "user-1",
+ assistant_message_id = None,
+ config = {
+ "model": "local-model",
+ "inferenceRequest": {"model": "local-model"},
+ "ragScope": None,
+ "instructions": "",
+ "question": "what happened?",
+ "websitePolicy": website_policy,
+ "budgets": {
+ "maxSteps": max_steps,
+ "maxSources": 5,
+ "modelTimeoutSeconds": 900,
+ "toolTimeoutSeconds": 10,
+ },
+ },
+ )
+ planned = research_db.set_plan("run-1", {"title": "Plan", "steps": plan_steps})
+ research_db.approve("run-1", planned["planRevision"], planned["planHash"])
+ return research_db.claim_next(supervisor.worker_id)
+
+
+def _run(
+ monkeypatch,
+ tool_results: list[str],
+ website_policy: dict | None = None,
+) -> dict:
+ from core import research_runs as worker
+
+ supervisor = worker.ResearchSupervisor(SimpleNamespace(state = SimpleNamespace(server_port = 1)))
+ plan_steps = [
+ {"title": f"Step {index}", "query": f"what happened {index}"}
+ for index in range(len(tool_results))
+ ]
+ claimed = _claimed_run(supervisor, plan_steps, len(tool_results), website_policy)
+ calls = {"n": 0}
+
+ def fake_execute_tool(name, arguments, **kwargs):
+ result = tool_results[min(calls["n"], len(tool_results) - 1)]
+ calls["n"] += 1
+ return result
+
+ async def fake_stream_completion(run, messages, **kwargs):
+ if kwargs.get("phase") == "synthesis":
+ return REPORT, "", "stop", None
+ # Unparseable, so every step falls back to the next unused plan seed.
+ return "not json", "", "stop", None
+
+ monkeypatch.setattr(worker, "execute_tool", fake_execute_tool)
+ monkeypatch.setattr(supervisor, "_stream_completion", fake_stream_completion)
+ asyncio.run(supervisor._process(claimed))
+ return research_db.get_run("run-1")
+
+
+def test_a_run_whose_every_search_was_throttled_fails_with_the_search_error(
+ research_home, monkeypatch
+):
+ finished = _run(monkeypatch, [THROTTLED, THROTTLED])
+
+ assert finished["status"] == "failed"
+ assert "rate limiting this machine" in (finished["error"] or "")
+ assert not finished["report"]
+ assert not finished["sources"]
+
+
+def test_a_run_whose_every_search_matched_nothing_fails(research_home, monkeypatch):
+ finished = _run(monkeypatch, ["No results found."])
+
+ assert finished["status"] == "failed"
+ assert not finished["report"]
+
+
+def test_the_failure_reaches_the_chat_message(research_home, monkeypatch):
+ _run(monkeypatch, [THROTTLED])
+
+ messages = studio_db.list_chat_messages("thread-1")
+ assistant = [message for message in messages if message["role"] == "assistant"][-1]
+ assert assistant["metadata"]["researchStatus"] == "failed"
+ assert "rate limiting this machine" in str(assistant["content"])
+
+
+def test_a_partially_failed_run_still_completes(research_home, monkeypatch):
+ finished = _run(monkeypatch, [FOUND, THROTTLED])
+
+ assert finished["status"] == "completed"
+ assert REPORT in finished["report"]
+ assert len(finished["sources"]) == 1
+
+
+def test_a_step_that_found_results_completes_the_run_even_with_no_citable_source(
+ research_home, monkeypatch
+):
+ """A sweep the website policy filters down to nothing catalogable still gathered the
+ evidence in its snippets, and its step is recorded completed."""
+ finished = _run(monkeypatch, [FOUND], {"allowedDomains": ["allowed.test"]})
+
+ assert finished["status"] == "completed"
+ assert not finished["sources"]
+ assert REPORT in finished["report"]
+
+
+ ("restored_result", "expected_status"),
+ [
+ ({"excerpt": "It happened on a Tuesday."}, "completed"),
+ ({}, "failed"),
+ ],
+)
+def test_a_resumed_step_counts_only_if_its_evidence_survived(
+ research_home, monkeypatch, restored_result, expected_status
+):
+ from core import research_runs as worker
+
+ supervisor = worker.ResearchSupervisor(SimpleNamespace(state = SimpleNamespace(server_port = 1)))
+ plan_steps = [
+ {"title": "One", "query": "what happened 0"},
+ {"title": "Two", "query": "what happened 1"},
+ ]
+ _claimed_run(supervisor, plan_steps, 2, None)
+ research_db.upsert_execution_step(
+ "run-1",
+ 0,
+ "One",
+ "what happened 0",
+ "completed",
+ {"action": "search", "input": "what happened 0", "sourceCount": 0, **restored_result},
+ supervisor.worker_id,
+ )
+ conn = studio_db.get_connection()
+ conn.execute(
+ "UPDATE research_runs SET lease_owner = NULL, lease_expires_at = 0 WHERE id = 'run-1'"
+ )
+ conn.commit()
+ conn.close()
+ resumed = research_db.claim_next(supervisor.worker_id)
+ assert resumed["claimedFromStatus"] == "running"
+
+ async def fake_stream_completion(run, messages, **kwargs):
+ if kwargs.get("phase") == "synthesis":
+ return REPORT, "", "stop", None
+ return "not json", "", "stop", None
+
+ monkeypatch.setattr(worker, "execute_tool", lambda *args, **kwargs: THROTTLED)
+ monkeypatch.setattr(supervisor, "_stream_completion", fake_stream_completion)
+ asyncio.run(supervisor._process(resumed))
+
+ assert research_db.get_run("run-1")["status"] == expected_status
diff --git a/studio/backend/tests/test_research_synthesis_recovery.py b/studio/backend/tests/test_research_synthesis_recovery.py
index 2e06215cbc5..57a06cc9b20 100644
--- a/studio/backend/tests/test_research_synthesis_recovery.py
+++ b/studio/backend/tests/test_research_synthesis_recovery.py
@@ -16,6 +16,12 @@ from storage import studio_db
FIRST_DRAFT = "## Findings\n\n" + ("The evidence says a great deal. " * 120).strip()
SHORTER_DRAFT = "## Findings\n\nToo little."
+# One gathered source: a run that gathers nothing fails before synthesis, which is not what
+# these tests are about.
+SEARCH_RESULT = (
+ "Title: What happened\nURL: https://example.test/what-happened\n"
+ "Snippet: It happened on a Tuesday."
+)
@pytest.fixture
@@ -40,6 +46,9 @@ def research_home(tmp_path, monkeypatch):
"createdAt": 2,
}
)
+ from core import research_runs as worker
+
+ monkeypatch.setattr(worker, "execute_tool", lambda *args, **kwargs: SEARCH_RESULT)
return tmp_path
@@ -74,7 +83,9 @@ def _claimed_run(supervisor, external: bool = False) -> dict:
},
},
)
- planned = research_db.set_plan("run-1", {"title": "Plan", "steps": []})
+ planned = research_db.set_plan(
+ "run-1", {"title": "Plan", "steps": [{"title": "Look it up", "query": "what happened"}]}
+ )
research_db.approve("run-1", planned["planRevision"], planned["planHash"])
return research_db.claim_next(supervisor.worker_id)
@@ -93,7 +104,7 @@ def _run_synthesis(monkeypatch, *, synthesis, recovery) -> dict:
return synthesis
if phase == "synthesis_recovery":
return recovery
- # Unparseable, and an empty plan has no seed action, so the step loop breaks.
+ # Unparseable, so the step falls back to the plan's one seed action.
return "not json", "", "stop", None
monkeypatch.setattr(supervisor, "_stream_completion", fake_stream_completion)