Skip to content

Automate browser based workflows with AI

fix(SKY-11238): emit best-effort task_v2 deliverable on budget exhaustion (#6778)

Trevor Cheung86ca44670146
diff --git a/skyvern/forge/prompts/skyvern/task_v2_summary.j2 b/skyvern/forge/prompts/skyvern/task_v2_summary.j2
index 5dfb0ac29..62e2b4e92 100644
--- a/skyvern/forge/prompts/skyvern/task_v2_summary.j2
+++ b/skyvern/forge/prompts/skyvern/task_v2_summary.j2
@@ -1,8 +1,11 @@
-The AI assistant has helped the user achieve the user goal in the web.
+{% if is_partial %}The AI assistant attempted the user goal but is stopping before fully completing it (it reached its step or planning-iteration budget). Produce the best possible version of the deliverable the user asked for, synthesized from all the data gathered so far.{% else %}The AI assistant has helped the user achieve the user goal in the web.{% endif %}
Given the user goal, the latest screenshot of the page, and the mini tasks completed along the way, produce the final answer to the user goal — the deliverable the user actually asked for — plus any structured data, in JSON format.
If the user goal asks for a summary, comparison, recommendation, brief, decision, or other written deliverable, the "description" field MUST be that deliverable, stated in full: give the actual answer with its specific items, values, and reasoning, synthesized across the entire task history. Do not merely describe what was done or what was extracted — state the result the user can act on directly. Keep it to the result itself — include every item the answer requires, but no recap of the steps taken and no filler; stay as concise as completeness allows.
+Match the output structure the goal explicitly asks for: if it names sections, headings, or questions to address, include each one as its own labeled part — do not merge, drop, or renumber them; if it asks for an exact number of items, rows, or options, return that many when the gathered data supports it (never pad, repeat, or invent entries to reach a count); if it asks for a recommendation or decision, state the actual choice, not just the options.
+{% if is_partial %}Some required parts may be missing because the run stopped early. Produce the deliverable for every part you have data for, and for any part still missing or unverified, include it with an explicit note that it could not be completed — never silently omit a requested section, item, or row.
+{% endif %}
Reply in JSON format with the following keys:
{
"description": str, // The final answer / deliverable for the user goal, stated in full — the recommendation, comparison, summary, or the extracted information itself, synthesized across the entire task history. Not a recap of the steps taken.
diff --git a/skyvern/services/task_v2_service.py b/skyvern/services/task_v2_service.py
index b12cadf24..06910ed4c 100644
--- a/skyvern/services/task_v2_service.py
+++ b/skyvern/services/task_v2_service.py
@@ -1247,12 +1247,17 @@ async def run_task_v2_helper(
failure_category_source="llm" if llm_failure_categories else "code_level",
failure_category_path="v2_max_steps",
)
+ summary_description, summarized_output = await _best_effort_failure_deliverable(
+ task_v2, task_history, context
+ )
task_v2 = await mark_task_v2_as_failed(
task_v2_id=task_v2_id,
workflow_run_id=workflow_run_id,
failure_reason=full_failure_reason,
organization_id=organization_id,
failure_category=failure_category,
+ summary=summary_description,
+ output=summarized_output,
)
return workflow, workflow_run, task_v2
else:
@@ -1279,12 +1284,15 @@ async def run_task_v2_helper(
failure_category_source="code_level",
failure_category_path="v2_max_iterations",
)
+ summary_description, summarized_output = await _best_effort_failure_deliverable(task_v2, task_history, context)
task_v2 = await mark_task_v2_as_failed(
task_v2_id=task_v2_id,
workflow_run_id=workflow_run_id,
failure_reason=max_iterations_failure_reason,
organization_id=organization_id,
failure_category=max_iterations_failure_category,
+ summary=summary_description,
+ output=summarized_output,
)
return workflow, workflow_run, task_v2
@@ -1829,11 +1837,15 @@ async def mark_task_v2_as_failed(
failure_reason: str | None = None,
organization_id: str | None = None,
failure_category: list[dict] | None = None,
+ summary: str | None = None,
+ output: dict[str, Any] | None = None,
) -> TaskV2:
task_v2 = await _update_task_v2_status(
task_v2_id,
organization_id=organization_id,
status=TaskV2Status.failed,
+ summary=summary,
+ output=output,
failure_category=failure_category,
)
if workflow_run_id:
@@ -2141,12 +2153,14 @@ async def _persist_completion_tab_screenshots(browser_state: BrowserState, thoug
return captured
-async def _summarize_task_v2(
+async def _generate_task_v2_deliverable(
task_v2: TaskV2,
task_history: list[dict],
context: SkyvernContext,
screenshots: list[bytes] | None = None,
-) -> TaskV2:
+ is_partial: bool = False,
+) -> tuple[str | None, Any]:
+ # is_partial=True renders the best-effort variant of the summary prompt; does not change task status.
thought = await app.DATABASE.observer.create_thought(
task_v2_id=task_v2.observer_cruise_id,
organization_id=task_v2.organization_id,
@@ -2162,6 +2176,7 @@ async def _summarize_task_v2(
user_goal=task_v2.prompt,
task_history=task_history,
extracted_information_schema=task_v2.extracted_information_schema,
+ is_partial=is_partial,
local_datetime=datetime.now(context.tz_info).isoformat(),
)
task_v2_summary_resp = await app.LLM_API_HANDLER(
@@ -2171,7 +2186,7 @@ async def _summarize_task_v2(
prompt_name="task_v2_summary",
system_prompt=task_v2.workflow_system_prompt,
)
- LOG.info("Task v2 summary response", task_v2_summary_resp=task_v2_summary_resp)
+ LOG.info("Task v2 summary response", task_v2_summary_resp=task_v2_summary_resp, is_partial=is_partial)
summary_description = task_v2_summary_resp.get("description")
summarized_output = task_v2_summary_resp.get("output")
@@ -2181,7 +2196,49 @@ async def _summarize_task_v2(
thought=summary_description,
output=task_v2_summary_resp,
)
+ return summary_description, summarized_output
+
+
+async def _best_effort_failure_deliverable(
+ task_v2: TaskV2,
+ task_history: list[dict],
+ context: SkyvernContext,
+) -> tuple[str | None, Any]:
+ # Never raises: deliverable synthesis must not interfere with terminal failure handling.
+ # Screenshots are skipped — synthesized from task_history; the failure paths already screenshot.
+ if not task_history:
+ return None, None
+ try:
+ return await _generate_task_v2_deliverable(
+ task_v2=task_v2,
+ task_history=task_history,
+ context=context,
+ screenshots=None,
+ is_partial=True,
+ )
+ except Exception:
+ LOG.warning(
+ "Failed to synthesize best-effort deliverable for failing task v2",
+ exc_info=True,
+ task_v2_id=task_v2.observer_cruise_id,
+ workflow_run_id=task_v2.workflow_run_id,
+ )
+ return None, None
+
+async def _summarize_task_v2(
+ task_v2: TaskV2,
+ task_history: list[dict],
+ context: SkyvernContext,
+ screenshots: list[bytes] | None = None,
+) -> TaskV2:
+ summary_description, summarized_output = await _generate_task_v2_deliverable(
+ task_v2=task_v2,
+ task_history=task_history,
+ context=context,
+ screenshots=screenshots,
+ is_partial=False,
+ )
return await mark_task_v2_as_completed(
task_v2_id=task_v2.observer_cruise_id,
workflow_run_id=task_v2.workflow_run_id,
diff --git a/tests/unit/test_task_v2_failure_deliverable.py b/tests/unit/test_task_v2_failure_deliverable.py
new file mode 100644
index 000000000..f3d3232f5
--- /dev/null
+++ b/tests/unit/test_task_v2_failure_deliverable.py
@@ -0,0 +1,122 @@
+"""task_v2 emits a best-effort deliverable from gathered data when it fails on budget exhaustion (SKY-11238)."""
+
+from __future__ import annotations
+
+import pytest
+
+from skyvern.forge.prompts import prompt_engine
+from skyvern.services import task_v2_service
+
+
+class _FakeTaskV2:
+ observer_cruise_id = "tsk_v2_test"
+ organization_id = "o_test"
+ workflow_run_id = "wr_test"
+ workflow_id = "w_test"
+ workflow_permanent_id = "wpid_test"
+ url = "https://example.com"
+ prompt = "do the thing"
+ extracted_information_schema = None
+ workflow_system_prompt = None
+
+
+async def test_best_effort_deliverable_skips_when_no_history_gathered() -> None:
+ # No data gathered => nothing to synthesize, and no LLM call should be attempted.
+ summary, output = await task_v2_service._best_effort_failure_deliverable(
+ _FakeTaskV2(), task_history=[], context=object()
+ )
+ assert summary is None
+ assert output is None
+
+
+async def test_best_effort_deliverable_never_raises(monkeypatch: pytest.MonkeyPatch) -> None:
+ async def _boom(**kwargs: object) -> tuple[str | None, dict | None]:
+ raise RuntimeError("llm down")
+
+ monkeypatch.setattr(task_v2_service, "_generate_task_v2_deliverable", _boom)
+ # Terminal failure handling must complete even if deliverable synthesis blows up.
+ summary, output = await task_v2_service._best_effort_failure_deliverable(
+ _FakeTaskV2(),
+ task_history=[{"type": "extract", "status": "completed"}],
+ context=object(),
+ )
+ assert summary is None
+ assert output is None
+
+
+async def test_best_effort_deliverable_requests_partial_and_returns_output_verbatim(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ captured: dict[str, object] = {}
+
+ async def _capture(
+ *, task_v2: object, task_history: list, context: object, screenshots: object, is_partial: bool
+ ) -> tuple[str | None, object]:
+ captured["is_partial"] = is_partial
+ captured["history_len"] = len(task_history)
+ captured["screenshots"] = screenshots
+ return "best effort memo", {"rows": 7}
+
+ monkeypatch.setattr(task_v2_service, "_generate_task_v2_deliverable", _capture)
+ history = [{"type": "extract", "status": "completed", "extracted_data": {"a": 1}}]
+ summary, output = await task_v2_service._best_effort_failure_deliverable(
+ _FakeTaskV2(), task_history=history, context=object()
+ )
+ assert summary == "best effort memo"
+ # Output is returned exactly as the model produced it -- no metadata injected into the user's schema.
+ assert output == {"rows": 7}
+ assert captured["is_partial"] is True
+ assert captured["history_len"] == 1
+ # Failure path skips screenshots (synthesizes from history only) to avoid a redundant capture.
+ assert captured["screenshots"] is None
+
+
+def test_summary_prompt_enforces_structure_and_partial_modes() -> None:
+ common = dict(
+ user_goal="Memo with sections A, B, C.",
+ task_history=[{"type": "extract", "status": "completed"}],
+ extracted_information_schema=None,
+ local_datetime="2026-06-19T12:00:00",
+ )
+ full = prompt_engine.load_prompt("task_v2_summary", is_partial=False, **common)
+ partial = prompt_engine.load_prompt("task_v2_summary", is_partial=True, **common)
+ # Structure-fidelity guidance is always present (helps merged-section / wrong-count near-misses).
+ assert "do not merge, drop, or renumber" in full
+ assert "do not merge, drop, or renumber" in partial
+ # Partial guidance only when the run stopped early.
+ assert "could not be completed" not in full
+ assert "could not be completed" in partial
+ # Default opening is byte-for-byte unchanged for the happy path.
+ assert full.startswith("The AI assistant has helped the user achieve the user goal in the web.")
+
+
+async def test_mark_task_v2_as_failed_threads_summary_and_output(monkeypatch: pytest.MonkeyPatch) -> None:
+ captured: dict[str, object] = {}
+
+ async def _fake_update(task_v2_id: str, **kwargs: object) -> _FakeTaskV2:
+ captured["task_v2_id"] = task_v2_id
+ captured.update(kwargs)
+ return _FakeTaskV2()
+
+ async def _noop_webhook(task_v2: object) -> None:
+ return None
+
+ monkeypatch.setattr(task_v2_service, "_update_task_v2_status", _fake_update)
+ monkeypatch.setattr(task_v2_service, "send_task_v2_webhook", _noop_webhook)
+ # No workflow_run_id => the workflow-run failure side effect is skipped; this exercises the
+ # persistence wiring that carries the best-effort deliverable onto a failed run.
+ result = await task_v2_service.mark_task_v2_as_failed(
+ task_v2_id="tsk_v2_x",
+ organization_id="o_x",
+ failure_reason="ran out of steps",
+ summary="best-effort memo",
+ output={"a": 1},
+ )
+ assert isinstance(result, _FakeTaskV2)
+ assert captured["status"] == task_v2_service.TaskV2Status.failed
+ assert captured["summary"] == "best-effort memo"
+ assert captured["output"] == {"a": 1}