Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Recover Task v2 summary deliverable when the LLM skips the response envelope (SKY-14004) (#8203)
Suchintan4a62365f51a8
diff --git a/skyvern/forge/prompts/skyvern/task_v2_summary.j2 b/skyvern/forge/prompts/skyvern/task_v2_summary.j2index 589e909b1..22901b99e 100644--- a/skyvern/forge/prompts/skyvern/task_v2_summary.j2+++ b/skyvern/forge/prompts/skyvern/task_v2_summary.j2@@ -12,6 +12,8 @@ Reply in JSON format with the following keys:"output": json, // Structured data related to the user goal if any.{% if extracted_information_schema %} Use the data schema for the output field.{% endif %}}+IMPORTANT: Reply with exactly this two-key envelope. If the user goal defines its own output format or JSON schema, put that entire JSON inside the "output" key — never at the top level of your reply.+User goal:```{{ user_goal }}diff --git a/skyvern/services/task_v2_service.py b/skyvern/services/task_v2_service.pyindex dddc62ae8..601478b6f 100644--- a/skyvern/services/task_v2_service.py+++ b/skyvern/services/task_v2_service.py@@ -2563,6 +2563,49 @@ async def _persist_completion_tab_screenshots(browser_state: BrowserState, thougreturn captured+def _parse_task_v2_summary_response(resp: Any) -> tuple[str | None, Any]:+ """Extract (description, output) from a task_v2_summary reply, tolerating envelope+ non-compliance.++ The summary prompt requires a {"description", "output"} envelope, but when the user+ goal defines its own output contract the model sometimes follows that contract+ instead, emitting the deliverable at the top level of the reply. In that case the+ reply itself IS the output; dropping it loses the run's extracted_information+ (SKY-14004). The recovered payload keeps every data key — including a user-schema+ "description" — so a goal-defined contract survives verbatim; only null envelope+ slots are dropped.++ The LLM handler runs with force_dict=True, so non-dict replies cannot reach this+ parser in production; they are handled defensively.+ """+ if not isinstance(resp, dict):+ return None, None+ description = resp.get("description")+ if not isinstance(description, str):+ description = None+ output = resp.get("output")+ if output is None:+ # Data beyond a compliant envelope: any key besides "output", except a+ # "description" holding the envelope's string (or null) slot. A non-str,+ # non-null "description" is user data, not the envelope.+ has_payload = any(+ key != "description" or (value is not None and not isinstance(value, str))+ for key, value in resp.items()+ if key != "output"+ )+ if has_payload:+ output = {+ key: value+ for key, value in resp.items()+ if key != "output" and not (key == "description" and value is None)+ }+ LOG.info(+ "Recovered task_v2 summary output from non-envelope response",+ recovered_keys=sorted(output.keys()),+ )+ return description, output++async def _generate_task_v2_deliverable(task_v2: TaskV2,task_history: list[dict],@@ -2597,8 +2640,7 @@ async def _generate_task_v2_deliverable()LOG.info("Task v2 summary response", task_v2_summary_resp=task_v2_summary_resp, is_partial=is_partial)- summary_description = task_v2_summary_resp.get("description")- summarized_output = task_v2_summary_resp.get("output")+ summary_description, summarized_output = _parse_task_v2_summary_response(task_v2_summary_resp)await app.DATABASE.observer.update_thought(thought_id=thought.observer_thought_id,organization_id=task_v2.organization_id,diff --git a/tests/unit/test_task_v2_summary_envelope.py b/tests/unit/test_task_v2_summary_envelope.pynew file mode 100644index 000000000..52c962575--- /dev/null+++ b/tests/unit/test_task_v2_summary_envelope.py@@ -0,0 +1,147 @@+"""task_v2 summary parsing tolerates envelope non-compliance so the deliverable is never dropped (SKY-14004).++When the user goal defines its own output contract, the summary model sometimes emits the+deliverable at the top level of its reply instead of inside the {"description", "output"}+envelope. The parser must recover that payload; production runs completed with+extracted_information=null because it did not. The LLM handler runs with force_dict=True,+so only dict replies reach the parser in production — non-dict handling is defensive.+"""++from __future__ import annotations++from datetime import timezone+from types import SimpleNamespace++import pytest++from skyvern.forge.prompts import prompt_engine+from skyvern.services import task_v2_service+from skyvern.services.task_v2_service import _parse_task_v2_summary_response+++class _FakeTaskV2:+ observer_cruise_id = "tsk_v2_test"+ organization_id = "o_test"+ workflow_run_id = "wr_test"+ workflow_id = "w_test"+ workflow_permanent_id = "wpid_test"+ url = "https://example.com"+ prompt = "map the application form and emit the field inventory JSON"+ extracted_information_schema = None+ workflow_system_prompt = None+++# Shape observed in production for the empty-output runs: the model followed the user+# goal's own schema (top-level field inventory) instead of the envelope.+_NON_ENVELOPE_RESPONSE = {+ "status": "success",+ "language": "en",+ "otpCodeRequired": False,+ "notes": "Single-page form; required-ness read from asterisks and validation attributes.",+ "fields": [+ {"name": "First name", "type": "text", "required": True, "validation": "maxlength: 127"},+ {"name": "Resume", "type": "file", "required": True},+ ],+}+++def test_parser_keeps_envelope_verbatim() -> None:+ resp = {"description": "the memo", "output": {"rows": [1, 2]}}+ assert _parse_task_v2_summary_response(resp) == ("the memo", {"rows": [1, 2]})+++def test_parser_description_only_yields_no_output() -> None:+ # Legit "no structured data" reply must stay None -- recovery only fires when keys+ # outside the envelope exist.+ assert _parse_task_v2_summary_response({"description": "navigated and submitted"}) == (+ "navigated and submitted",+ None,+ )+ assert _parse_task_v2_summary_response({"description": "done", "output": None}) == ("done", None)+++def test_parser_recovers_top_level_payload() -> None:+ description, output = _parse_task_v2_summary_response(dict(_NON_ENVELOPE_RESPONSE))+ assert description is None+ # The whole non-envelope reply IS the deliverable.+ assert output == _NON_ENVELOPE_RESPONSE+++def test_parser_recovery_preserves_user_schema_description_key() -> None:+ # A goal-defined contract may itself use "description"; recovery must not strip it.+ resp = {"description": "Widget", "price": 10}+ description, output = _parse_task_v2_summary_response(resp)+ assert description == "Widget"+ assert output == {"description": "Widget", "price": 10}+++def test_parser_recovers_non_str_description_as_payload() -> None:+ # A non-str, non-null "description" is user data, not the envelope slot.+ assert _parse_task_v2_summary_response({"description": ["a", "b"]}) == (None, {"description": ["a", "b"]})+ # A null "description" alone carries no data.+ assert _parse_task_v2_summary_response({"description": None}) == (None, None)+ # Null envelope slots are dropped from a recovered payload.+ assert _parse_task_v2_summary_response({"description": None, "fields": [1]}) == (None, {"fields": [1]})+++def test_parser_recovers_stray_keys_next_to_null_output() -> None:+ resp = {"description": "the memo", "output": None, "fields": [{"name": "Email"}]}+ description, output = _parse_task_v2_summary_response(resp)+ assert description == "the memo"+ # Only the null envelope "output" slot is dropped; every data key survives.+ assert output == {"description": "the memo", "fields": [{"name": "Email"}]}+++def test_parser_tolerates_non_dict_and_non_str_description() -> None:+ # force_dict=True upstream makes non-dict unreachable in production; stay total anyway.+ assert _parse_task_v2_summary_response("just prose") == (None, None)+ assert _parse_task_v2_summary_response(None) == (None, None)+ assert _parse_task_v2_summary_response({}) == (None, None)+ # A non-str description must not reach the varchar summary column.+ description, output = _parse_task_v2_summary_response({"description": 42, "output": {"a": 1}})+ assert description is None+ assert output == {"a": 1}+++def test_summary_prompt_pins_envelope_over_user_goal_schema() -> None:+ common = dict(+ user_goal="Emit the field inventory as a top-level JSON object.",+ task_history=[{"type": "extract", "status": "completed"}],+ extracted_information_schema=None,+ local_datetime="2026-08-13T12:00:00",+ )+ full = prompt_engine.load_prompt("task_v2_summary", is_partial=False, **common)+ partial = prompt_engine.load_prompt("task_v2_summary", is_partial=True, **common)+ for rendered in (full, partial):+ assert 'put that entire JSON inside the "output" key' in rendered+++async def test_generate_deliverable_recovers_non_envelope_response(monkeypatch: pytest.MonkeyPatch) -> None:+ """End-to-end through _generate_task_v2_deliverable with the production failure shape."""+ thought_updates: dict[str, object] = {}++ class _FakeObserver:+ async def create_thought(self, **kwargs: object) -> SimpleNamespace:+ return SimpleNamespace(observer_thought_id="ot_test")++ async def update_thought(self, **kwargs: object) -> None:+ thought_updates.update(kwargs)++ monkeypatch.setattr(task_v2_service.app, "DATABASE", SimpleNamespace(observer=_FakeObserver()))++ async def _llm_handler(**kwargs: object) -> dict:+ return dict(_NON_ENVELOPE_RESPONSE)++ monkeypatch.setattr(task_v2_service, "_get_task_v2_llm_api_handler", lambda task_v2: _llm_handler)++ description, output = await task_v2_service._generate_task_v2_deliverable(+ task_v2=_FakeTaskV2(),+ task_history=[{"type": "extract", "status": "completed", "extracted_data": {"fields": []}}],+ context=SimpleNamespace(tz_info=timezone.utc),+ )+ assert description is None+ assert output == _NON_ENVELOPE_RESPONSE+ # The raw reply still lands on the summarization thought for auditability/backfill.+ assert thought_updates["output"] == _NON_ENVELOPE_RESPONSE+ assert thought_updates["thought"] is None