Skip to content

Automate browser based workflows with AI

fix(SKY-9350): unwrap raw REPLY envelope before chat render (#5728)

Andrew Neilson7f93210cde8f
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2
index 7f7a6a152..79001b58b 100644
--- a/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2
+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2
@@ -194,6 +194,8 @@ Note: The user works with a UI and is not aware of YAML. Avoid mentioning YAML i
MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments.
+Do NOT prefix the JSON with the action name. The output must START with `{` — no leading `REPLY`, `ASK_QUESTION`, `REPLACE_WORKFLOW`, "Option 1", "Option 2", or any other label outside the JSON.
+
Current datetime, ISO format:
```
{{ current_datetime }}
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-system.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-system.j2
index 695b4ef00..24c4ca358 100644
--- a/skyvern/forge/prompts/skyvern/workflow-copilot-system.j2
+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-system.j2
@@ -140,6 +140,8 @@ Use this when the user's request is ambiguous, missing critical information, or
MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes.
All YAML content must be properly escaped as JSON strings.
+Do NOT prefix the JSON with the action name. The output must START with `{` — no leading `REPLY`, `ASK_QUESTION`, `REPLACE_WORKFLOW`, "Option 1", "Option 2", or any other label outside the JSON.
+
Current datetime, ISO format:
```
{{ current_datetime }}
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 2dc7f069f..9f630377b 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -25,7 +25,7 @@ from pydantic import ValidationError
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.copilot.block_goal_wrapping import wrap_block_goals
-from skyvern.forge.sdk.copilot.context import AgentResult, CopilotContext, StructuredContext
+from skyvern.forge.sdk.copilot.context import COPILOT_RESPONSE_TYPES, AgentResult, CopilotContext, StructuredContext
from skyvern.forge.sdk.copilot.output_utils import extract_final_text, parse_final_response
from skyvern.forge.sdk.copilot.tracing_setup import _copilot_model_name, ensure_tracing_initialized, is_tracing_enabled
from skyvern.forge.sdk.schemas.persistent_browser_sessions import is_final_status
@@ -262,7 +262,7 @@ def _translate_to_agent_result(
user_response = action_data.get("user_response") or "Done."
resp_type = action_data.get("type", "REPLY")
- if resp_type not in ("REPLY", "ASK_QUESTION", "REPLACE_WORKFLOW"):
+ if resp_type not in COPILOT_RESPONSE_TYPES:
resp_type = "REPLY"
last_workflow = ctx.last_workflow
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 7d5a65786..f801c8cac 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -4,7 +4,7 @@ from __future__ import annotations
import re
from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, Any, Literal
+from typing import TYPE_CHECKING, Any, Literal, get_args
from pydantic import BaseModel, Field
@@ -12,6 +12,7 @@ from skyvern.forge.sdk.copilot.runtime import AgentContext
from skyvern.forge.sdk.workflow.models.workflow import Workflow
ResponseType = Literal["REPLY", "ASK_QUESTION", "REPLACE_WORKFLOW"]
+COPILOT_RESPONSE_TYPES: tuple[str, ...] = get_args(ResponseType)
if TYPE_CHECKING:
from skyvern.forge.sdk.copilot.narration import NarratorState
diff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.py
index 595e5b49e..5b90058a9 100644
--- a/skyvern/forge/sdk/copilot/output_utils.py
+++ b/skyvern/forge/sdk/copilot/output_utils.py
@@ -9,6 +9,8 @@ import re
from collections.abc import Iterator
from typing import TYPE_CHECKING, Any
+from skyvern.forge.sdk.copilot.context import COPILOT_RESPONSE_TYPES
+
if TYPE_CHECKING:
from agents.result import RunResultStreaming
@@ -38,15 +40,51 @@ def extract_final_text(result: RunResultStreaming) -> str:
return ""
+_TYPE_ALTERNATION = "|".join(COPILOT_RESPONSE_TYPES)
+_LEADING_LABEL_RE = re.compile(rf"^\s*({_TYPE_ALTERNATION})\s*[:,]?\s+", re.IGNORECASE)
+_USER_RESPONSE_VALUE_RE = re.compile(r'"user_response"\s*:\s*"((?:[^"\\]|\\.)*)"')
+_TYPE_VALUE_RE = re.compile(rf'"type"\s*:\s*"({_TYPE_ALTERNATION})"')
+
+
+def _try_loads_dict(text: str) -> dict[str, Any] | None:
+ # strict=False allows literal control characters in string values (SKY-9189)
+ try:
+ parsed = json.loads(text, strict=False)
+ except json.JSONDecodeError:
+ return None
+ return parsed if isinstance(parsed, dict) else None
+
+
+def _looks_like_envelope(parsed: dict[str, Any]) -> bool:
+ if "user_response" in parsed:
+ return True
+ # bare {"type": "object"} (a JSON schema in prose) is not an envelope
+ type_value = parsed.get("type")
+ return isinstance(type_value, str) and type_value.upper() in COPILOT_RESPONSE_TYPES
+
+
+def _text_looks_envelope_shaped(text: str) -> bool:
+ # require leading `{` so prose that merely quotes the field names (e.g.,
+ # "I see \"type\": \"REPLY\" but cannot find \"user_response\"") falls
+ # through to the plain-text tier instead of degrading to "Done."
+ return text.startswith("{") and '"user_response"' in text and bool(_TYPE_VALUE_RE.search(text))
+
+
+def _sniff_response_type(text: str) -> str:
+ # REPLACE_WORKFLOW is demoted to REPLY: recovery cannot extract a usable
+ # workflow_yaml, and announcing an update without one is worse than silent.
+ match = _TYPE_VALUE_RE.search(text)
+ if match and match.group(1).upper() == "ASK_QUESTION":
+ return "ASK_QUESTION"
+ return "REPLY"
+
+
def parse_final_response(text: str) -> dict[str, Any]:
- """Parse the agent's final JSON envelope, stripping markdown code fences.
-
- Tolerant of literal control characters (newlines/tabs/CR) inside string
- values. Some model outputs wrap long `user_response` prose across real
- newlines instead of `\\n` escapes; strict `json.loads` rejects those, and
- without this fallback the frontend renders the full raw JSON object as
- the user-visible reply (see SKY-9189 test-2 regression).
- """
+ """Parse the agent's final JSON envelope, tolerating markdown code fences,
+ leading action labels (``REPLY {...}``), prose preambles, and literal
+ control characters in string values. Falls back to regex-extracting
+ ``user_response`` from envelope-shaped text so a malformed envelope never
+ reaches the chat bubble."""
cleaned = text.strip()
for prefix in ("```json", "```"):
if cleaned.startswith(prefix):
@@ -56,13 +94,35 @@ def parse_final_response(text: str) -> dict[str, Any]:
cleaned = cleaned[:-3]
cleaned = cleaned.strip()
- for strict in (True, False):
- try:
- parsed = json.loads(cleaned, strict=strict)
- if isinstance(parsed, dict):
- return parsed
- except json.JSONDecodeError:
- continue
+ parsed = _try_loads_dict(cleaned)
+ if parsed is not None:
+ return parsed
+
+ label_stripped = _LEADING_LABEL_RE.sub("", cleaned, count=1)
+ if label_stripped != cleaned:
+ parsed = _try_loads_dict(label_stripped)
+ if parsed is not None:
+ return parsed
+
+ first = cleaned.find("{")
+ last = cleaned.rfind("}")
+ # skip when the slice equals the full string — _try_loads_dict above already tried it
+ if first != -1 and last > first and not (first == 0 and last == len(cleaned) - 1):
+ parsed = _try_loads_dict(cleaned[first : last + 1])
+ if parsed is not None and _looks_like_envelope(parsed):
+ return parsed
+
+ if _text_looks_envelope_shaped(cleaned):
+ sniffed_type = _sniff_response_type(cleaned)
+ match = _USER_RESPONSE_VALUE_RE.search(cleaned)
+ if match:
+ try:
+ value = json.loads(f'"{match.group(1)}"', strict=False)
+ except json.JSONDecodeError:
+ value = None
+ if isinstance(value, str):
+ return {"type": sniffed_type, "user_response": value}
+ return {"type": sniffed_type, "user_response": "Done."}
return {"type": "REPLY", "user_response": text}
diff --git a/tests/unit/test_copilot_output_utils.py b/tests/unit/test_copilot_output_utils.py
index 60b548563..e393e9fe4 100644
--- a/tests/unit/test_copilot_output_utils.py
+++ b/tests/unit/test_copilot_output_utils.py
@@ -441,3 +441,94 @@ class TestParseFinalResponse:
# A JSON array at top level is valid JSON but not a valid envelope.
parsed = parse_final_response("[1, 2, 3]")
assert parsed == {"type": "REPLY", "user_response": "[1, 2, 3]"}
+
+ def test_strips_leading_reply_label_before_parse(self) -> None:
+ envelope = 'REPLY\n{"type": "REPLY", "user_response": "ok"}'
+ parsed = parse_final_response(envelope)
+ assert parsed["type"] == "REPLY"
+ assert parsed["user_response"] == "ok"
+
+ def test_strips_leading_ask_question_label_with_colon(self) -> None:
+ envelope = 'ASK_QUESTION:\n{"type": "ASK_QUESTION", "user_response": "what date?"}'
+ parsed = parse_final_response(envelope)
+ assert parsed["type"] == "ASK_QUESTION"
+ assert parsed["user_response"] == "what date?"
+
+ def test_strips_leading_replace_workflow_label(self) -> None:
+ envelope = (
+ 'REPLACE_WORKFLOW {"type": "REPLACE_WORKFLOW", "user_response": "updated", "workflow_yaml": "title: x"}'
+ )
+ parsed = parse_final_response(envelope)
+ assert parsed["type"] == "REPLACE_WORKFLOW"
+ assert parsed["workflow_yaml"] == "title: x"
+
+ def test_extracts_json_after_prose_preamble(self) -> None:
+ envelope = 'Here\'s my response: {"type": "REPLY", "user_response": "ok"}'
+ parsed = parse_final_response(envelope)
+ assert parsed["type"] == "REPLY"
+ assert parsed["user_response"] == "ok"
+
+ def test_pass_b_rejects_non_envelope_dict_in_prose(self) -> None:
+ text = 'I cannot help with {"foo": "bar"}'
+ parsed = parse_final_response(text)
+ assert parsed == {"type": "REPLY", "user_response": text}
+
+ def test_pass_b_rejects_dict_with_unrecognized_type(self) -> None:
+ text = 'I cannot help with {"type": "object"}'
+ parsed = parse_final_response(text)
+ assert parsed == {"type": "REPLY", "user_response": text}
+
+ def test_recovery_tier_skipped_when_text_only_mentions_user_response(self) -> None:
+ text = 'I cannot find the "user_response" field in your input.'
+ parsed = parse_final_response(text)
+ assert parsed == {"type": "REPLY", "user_response": text}
+
+ def test_recovery_tier_skipped_when_prose_quotes_both_markers(self) -> None:
+ # Prose discussing the envelope format (both quoted `"type": "REPLY"`
+ # and `"user_response"` substrings present, no leading `{`) must not
+ # degrade to "Done." — the user's actual prose has to survive.
+ text = 'I see "type": "REPLY" mentioned, but cannot find "user_response" anywhere.'
+ parsed = parse_final_response(text)
+ assert parsed == {"type": "REPLY", "user_response": text}
+
+ def test_recovers_user_response_when_global_llm_context_malformed(self) -> None:
+ envelope = '{"type": "REPLY", "user_response": "the real answer", "global_llm_context": {"user_goal": "x",}}'
+ parsed = parse_final_response(envelope)
+ assert parsed["user_response"] == "the real answer"
+ assert parsed["type"] == "REPLY"
+
+ def test_recovers_user_response_with_escaped_quotes(self) -> None:
+ envelope = '{"type": "REPLY", "user_response": "she said \\"hi\\"", "global_llm_context": {bad}}'
+ parsed = parse_final_response(envelope)
+ assert parsed["user_response"] == 'she said "hi"'
+
+ def test_regex_recovery_tolerates_literal_newline_in_user_response_value(self) -> None:
+ envelope = '{"type": "REPLY", "user_response": "line one\nline two", "global_llm_context": {bad}}'
+ parsed = parse_final_response(envelope)
+ assert parsed["user_response"] == "line one\nline two"
+
+ def test_recovers_ask_question_type_when_recovering_user_response(self) -> None:
+ envelope = '{"type": "ASK_QUESTION", "user_response": "which account?", "global_llm_context": {bad}}'
+ parsed = parse_final_response(envelope)
+ assert parsed["type"] == "ASK_QUESTION"
+ assert parsed["user_response"] == "which account?"
+
+ def test_recovery_demotes_malformed_replace_workflow_to_reply(self) -> None:
+ # Recovery cannot extract workflow_yaml, so REPLACE_WORKFLOW would be
+ # unverified — demote to REPLY.
+ envelope = '{"type": "REPLACE_WORKFLOW", "user_response": "updated your workflow", "global_llm_context": {bad}}'
+ parsed = parse_final_response(envelope)
+ assert parsed["type"] == "REPLY"
+ assert parsed["user_response"] == "updated your workflow"
+
+ def test_envelope_shaped_unparseable_with_no_recoverable_user_response_returns_done(self) -> None:
+ envelope = '{"type": "REPLY", "user_response": "broken'
+ parsed = parse_final_response(envelope)
+ assert parsed["user_response"] == "Done."
+ assert parsed["type"] == "REPLY"
+ assert "broken" not in parsed["user_response"]
+
+ def test_non_envelope_unparseable_text_still_falls_back_to_text(self) -> None:
+ text = "I'm not sure how to help with that."
+ parsed = parse_final_response(text)
+ assert parsed == {"type": "REPLY", "user_response": text}