Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Fail agent runs that finish without producing output

Abi Raja91c400f44dec
diff --git a/backend/agent/engine.py b/backend/agent/engine.py
index 9ad1655e..11ead964 100644
--- a/backend/agent/engine.py
+++ b/backend/agent/engine.py
@@ -22,6 +22,19 @@ from config import GENERATION_MAX_COST_USD
from fs_logging.agent_runs import AgentRunRecorder
+class EmptyOutputError(Exception):
+ """Raised when a run finishes without producing any HTML.
+
+ Some models (observed: gemini-3.6-flash) occasionally run asset tools
+ and then stop without calling create_file. Treating that as success
+ poisons evals: the run looks green, diff mode skips it forever, and
+ the output file is empty. Raising makes it a normal, retryable failure.
+ """
+
+ def __init__(self) -> None:
+ super().__init__("Generation finished without producing any output.")
+
+
class BudgetExceededError(Exception):
"""Raised when a single generation exceeds the spend ceiling.
@@ -340,6 +353,8 @@ class AgentEngine:
)
try:
result = await self._run_with_session(session)
+ if not result:
+ raise EmptyOutputError()
if self.recorder is not None:
await self.recorder.record_run_end("completed", final_html=result)
return result
diff --git a/backend/tests/test_agent_engine.py b/backend/tests/test_agent_engine.py
index 7275201f..7a097adb 100644
--- a/backend/tests/test_agent_engine.py
+++ b/backend/tests/test_agent_engine.py
@@ -13,7 +13,9 @@ class NoToolProviderSession:
self.closed = False
async def stream_turn(self, on_event: EventSink) -> ProviderTurn:
- return ProviderTurn(assistant_text="", tool_calls=[])
+ # Non-empty output so the engine's EmptyOutputError guard stays quiet;
+ # this test only cares about tool-availability wiring.
+ return ProviderTurn(assistant_text="<html>ok</html>", tool_calls=[])
async def append_tool_results(
self,
diff --git a/backend/tests/test_agent_runs.py b/backend/tests/test_agent_runs.py
index 813e5452..46f62660 100644
--- a/backend/tests/test_agent_runs.py
+++ b/backend/tests/test_agent_runs.py
@@ -627,6 +627,41 @@ async def test_eval_runner_does_not_retry_budget_abort(
assert calls["count"] == 1 # no retries
+async def test_engine_fails_empty_output_runs(
+ logs_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ from agent.engine import EmptyOutputError
+ from agent.providers.base import ProviderTurn
+
+ class _NoOutputSession:
+ def total_cost_usd(self) -> Optional[float]:
+ return None
+
+ async def stream_turn(self, on_event: Any) -> Any:
+ # The observed gemini-3.6-flash failure mode: a final turn with
+ # no tool calls and no text at all.
+ return ProviderTurn(assistant_text="", tool_calls=[])
+
+ async def append_tool_results(self, turn: Any, executed: Any) -> None:
+ return None
+
+ async def close(self) -> None:
+ return None
+
+ monkeypatch.setattr(
+ "agent.engine.create_provider_session", lambda **kwargs: _NoOutputSession()
+ )
+ recorder = _make_recorder()
+ engine = _make_engine(recorder)
+ with pytest.raises(EmptyOutputError):
+ await engine.run(Llm.GPT_5_5_HIGH, [])
+
+ events = _read_events(recorder)
+ assert events[-1]["status"] == "failed"
+ assert "without producing" in events[-1]["error"]
+
+
@pytest.mark.asyncio
async def test_engine_records_failed_run_and_reraises(
logs_path: Path, monkeypatch: pytest.MonkeyPatch