Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-9350): unwrap raw REPLY envelope before chat render (#5728)
Andrew Neilson7f93210cde8f
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2index 7f7a6a152..79001b58b 100644--- a/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2@@ -194,6 +194,8 @@ Note: The user works with a UI and is not aware of YAML. Avoid mentioning YAML iMAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments.+Do NOT prefix the JSON with the action name. The output must START with `{` — no leading `REPLY`, `ASK_QUESTION`, `REPLACE_WORKFLOW`, "Option 1", "Option 2", or any other label outside the JSON.+Current datetime, ISO format:```{{ current_datetime }}diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-system.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-system.j2index 695b4ef00..24c4ca358 100644--- a/skyvern/forge/prompts/skyvern/workflow-copilot-system.j2+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-system.j2@@ -140,6 +140,8 @@ Use this when the user's request is ambiguous, missing critical information, orMAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes.All YAML content must be properly escaped as JSON strings.+Do NOT prefix the JSON with the action name. The output must START with `{` — no leading `REPLY`, `ASK_QUESTION`, `REPLACE_WORKFLOW`, "Option 1", "Option 2", or any other label outside the JSON.+Current datetime, ISO format:```{{ current_datetime }}diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 2dc7f069f..9f630377b 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -25,7 +25,7 @@ from pydantic import ValidationErrorfrom skyvern.forge import appfrom skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.copilot.block_goal_wrapping import wrap_block_goals-from skyvern.forge.sdk.copilot.context import AgentResult, CopilotContext, StructuredContext+from skyvern.forge.sdk.copilot.context import COPILOT_RESPONSE_TYPES, AgentResult, CopilotContext, StructuredContextfrom skyvern.forge.sdk.copilot.output_utils import extract_final_text, parse_final_responsefrom skyvern.forge.sdk.copilot.tracing_setup import _copilot_model_name, ensure_tracing_initialized, is_tracing_enabledfrom skyvern.forge.sdk.schemas.persistent_browser_sessions import is_final_status@@ -262,7 +262,7 @@ def _translate_to_agent_result(user_response = action_data.get("user_response") or "Done."resp_type = action_data.get("type", "REPLY")- if resp_type not in ("REPLY", "ASK_QUESTION", "REPLACE_WORKFLOW"):+ if resp_type not in COPILOT_RESPONSE_TYPES:resp_type = "REPLY"last_workflow = ctx.last_workflowdiff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex 7d5a65786..f801c8cac 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -4,7 +4,7 @@ from __future__ import annotationsimport refrom dataclasses import dataclass, field-from typing import TYPE_CHECKING, Any, Literal+from typing import TYPE_CHECKING, Any, Literal, get_argsfrom pydantic import BaseModel, Field@@ -12,6 +12,7 @@ from skyvern.forge.sdk.copilot.runtime import AgentContextfrom skyvern.forge.sdk.workflow.models.workflow import WorkflowResponseType = Literal["REPLY", "ASK_QUESTION", "REPLACE_WORKFLOW"]+COPILOT_RESPONSE_TYPES: tuple[str, ...] = get_args(ResponseType)if TYPE_CHECKING:from skyvern.forge.sdk.copilot.narration import NarratorStatediff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.pyindex 595e5b49e..5b90058a9 100644--- a/skyvern/forge/sdk/copilot/output_utils.py+++ b/skyvern/forge/sdk/copilot/output_utils.py@@ -9,6 +9,8 @@ import refrom collections.abc import Iteratorfrom typing import TYPE_CHECKING, Any+from skyvern.forge.sdk.copilot.context import COPILOT_RESPONSE_TYPES+if TYPE_CHECKING:from agents.result import RunResultStreaming@@ -38,15 +40,51 @@ def extract_final_text(result: RunResultStreaming) -> str:return ""+_TYPE_ALTERNATION = "|".join(COPILOT_RESPONSE_TYPES)+_LEADING_LABEL_RE = re.compile(rf"^\s*({_TYPE_ALTERNATION})\s*[:,]?\s+", re.IGNORECASE)+_USER_RESPONSE_VALUE_RE = re.compile(r'"user_response"\s*:\s*"((?:[^"\\]|\\.)*)"')+_TYPE_VALUE_RE = re.compile(rf'"type"\s*:\s*"({_TYPE_ALTERNATION})"')+++def _try_loads_dict(text: str) -> dict[str, Any] | None:+ # strict=False allows literal control characters in string values (SKY-9189)+ try:+ parsed = json.loads(text, strict=False)+ except json.JSONDecodeError:+ return None+ return parsed if isinstance(parsed, dict) else None+++def _looks_like_envelope(parsed: dict[str, Any]) -> bool:+ if "user_response" in parsed:+ return True+ # bare {"type": "object"} (a JSON schema in prose) is not an envelope+ type_value = parsed.get("type")+ return isinstance(type_value, str) and type_value.upper() in COPILOT_RESPONSE_TYPES+++def _text_looks_envelope_shaped(text: str) -> bool:+ # require leading `{` so prose that merely quotes the field names (e.g.,+ # "I see \"type\": \"REPLY\" but cannot find \"user_response\"") falls+ # through to the plain-text tier instead of degrading to "Done."+ return text.startswith("{") and '"user_response"' in text and bool(_TYPE_VALUE_RE.search(text))+++def _sniff_response_type(text: str) -> str:+ # REPLACE_WORKFLOW is demoted to REPLY: recovery cannot extract a usable+ # workflow_yaml, and announcing an update without one is worse than silent.+ match = _TYPE_VALUE_RE.search(text)+ if match and match.group(1).upper() == "ASK_QUESTION":+ return "ASK_QUESTION"+ return "REPLY"++def parse_final_response(text: str) -> dict[str, Any]:- """Parse the agent's final JSON envelope, stripping markdown code fences.-- Tolerant of literal control characters (newlines/tabs/CR) inside string- values. Some model outputs wrap long `user_response` prose across real- newlines instead of `\\n` escapes; strict `json.loads` rejects those, and- without this fallback the frontend renders the full raw JSON object as- the user-visible reply (see SKY-9189 test-2 regression).- """+ """Parse the agent's final JSON envelope, tolerating markdown code fences,+ leading action labels (``REPLY {...}``), prose preambles, and literal+ control characters in string values. Falls back to regex-extracting+ ``user_response`` from envelope-shaped text so a malformed envelope never+ reaches the chat bubble."""cleaned = text.strip()for prefix in ("```json", "```"):if cleaned.startswith(prefix):@@ -56,13 +94,35 @@ def parse_final_response(text: str) -> dict[str, Any]:cleaned = cleaned[:-3]cleaned = cleaned.strip()- for strict in (True, False):- try:- parsed = json.loads(cleaned, strict=strict)- if isinstance(parsed, dict):- return parsed- except json.JSONDecodeError:- continue+ parsed = _try_loads_dict(cleaned)+ if parsed is not None:+ return parsed++ label_stripped = _LEADING_LABEL_RE.sub("", cleaned, count=1)+ if label_stripped != cleaned:+ parsed = _try_loads_dict(label_stripped)+ if parsed is not None:+ return parsed++ first = cleaned.find("{")+ last = cleaned.rfind("}")+ # skip when the slice equals the full string — _try_loads_dict above already tried it+ if first != -1 and last > first and not (first == 0 and last == len(cleaned) - 1):+ parsed = _try_loads_dict(cleaned[first : last + 1])+ if parsed is not None and _looks_like_envelope(parsed):+ return parsed++ if _text_looks_envelope_shaped(cleaned):+ sniffed_type = _sniff_response_type(cleaned)+ match = _USER_RESPONSE_VALUE_RE.search(cleaned)+ if match:+ try:+ value = json.loads(f'"{match.group(1)}"', strict=False)+ except json.JSONDecodeError:+ value = None+ if isinstance(value, str):+ return {"type": sniffed_type, "user_response": value}+ return {"type": sniffed_type, "user_response": "Done."}return {"type": "REPLY", "user_response": text}diff --git a/tests/unit/test_copilot_output_utils.py b/tests/unit/test_copilot_output_utils.pyindex 60b548563..e393e9fe4 100644--- a/tests/unit/test_copilot_output_utils.py+++ b/tests/unit/test_copilot_output_utils.py@@ -441,3 +441,94 @@ class TestParseFinalResponse:# A JSON array at top level is valid JSON but not a valid envelope.parsed = parse_final_response("[1, 2, 3]")assert parsed == {"type": "REPLY", "user_response": "[1, 2, 3]"}++ def test_strips_leading_reply_label_before_parse(self) -> None:+ envelope = 'REPLY\n{"type": "REPLY", "user_response": "ok"}'+ parsed = parse_final_response(envelope)+ assert parsed["type"] == "REPLY"+ assert parsed["user_response"] == "ok"++ def test_strips_leading_ask_question_label_with_colon(self) -> None:+ envelope = 'ASK_QUESTION:\n{"type": "ASK_QUESTION", "user_response": "what date?"}'+ parsed = parse_final_response(envelope)+ assert parsed["type"] == "ASK_QUESTION"+ assert parsed["user_response"] == "what date?"++ def test_strips_leading_replace_workflow_label(self) -> None:+ envelope = (+ 'REPLACE_WORKFLOW {"type": "REPLACE_WORKFLOW", "user_response": "updated", "workflow_yaml": "title: x"}'+ )+ parsed = parse_final_response(envelope)+ assert parsed["type"] == "REPLACE_WORKFLOW"+ assert parsed["workflow_yaml"] == "title: x"++ def test_extracts_json_after_prose_preamble(self) -> None:+ envelope = 'Here\'s my response: {"type": "REPLY", "user_response": "ok"}'+ parsed = parse_final_response(envelope)+ assert parsed["type"] == "REPLY"+ assert parsed["user_response"] == "ok"++ def test_pass_b_rejects_non_envelope_dict_in_prose(self) -> None:+ text = 'I cannot help with {"foo": "bar"}'+ parsed = parse_final_response(text)+ assert parsed == {"type": "REPLY", "user_response": text}++ def test_pass_b_rejects_dict_with_unrecognized_type(self) -> None:+ text = 'I cannot help with {"type": "object"}'+ parsed = parse_final_response(text)+ assert parsed == {"type": "REPLY", "user_response": text}++ def test_recovery_tier_skipped_when_text_only_mentions_user_response(self) -> None:+ text = 'I cannot find the "user_response" field in your input.'+ parsed = parse_final_response(text)+ assert parsed == {"type": "REPLY", "user_response": text}++ def test_recovery_tier_skipped_when_prose_quotes_both_markers(self) -> None:+ # Prose discussing the envelope format (both quoted `"type": "REPLY"`+ # and `"user_response"` substrings present, no leading `{`) must not+ # degrade to "Done." — the user's actual prose has to survive.+ text = 'I see "type": "REPLY" mentioned, but cannot find "user_response" anywhere.'+ parsed = parse_final_response(text)+ assert parsed == {"type": "REPLY", "user_response": text}++ def test_recovers_user_response_when_global_llm_context_malformed(self) -> None:+ envelope = '{"type": "REPLY", "user_response": "the real answer", "global_llm_context": {"user_goal": "x",}}'+ parsed = parse_final_response(envelope)+ assert parsed["user_response"] == "the real answer"+ assert parsed["type"] == "REPLY"++ def test_recovers_user_response_with_escaped_quotes(self) -> None:+ envelope = '{"type": "REPLY", "user_response": "she said \\"hi\\"", "global_llm_context": {bad}}'+ parsed = parse_final_response(envelope)+ assert parsed["user_response"] == 'she said "hi"'++ def test_regex_recovery_tolerates_literal_newline_in_user_response_value(self) -> None:+ envelope = '{"type": "REPLY", "user_response": "line one\nline two", "global_llm_context": {bad}}'+ parsed = parse_final_response(envelope)+ assert parsed["user_response"] == "line one\nline two"++ def test_recovers_ask_question_type_when_recovering_user_response(self) -> None:+ envelope = '{"type": "ASK_QUESTION", "user_response": "which account?", "global_llm_context": {bad}}'+ parsed = parse_final_response(envelope)+ assert parsed["type"] == "ASK_QUESTION"+ assert parsed["user_response"] == "which account?"++ def test_recovery_demotes_malformed_replace_workflow_to_reply(self) -> None:+ # Recovery cannot extract workflow_yaml, so REPLACE_WORKFLOW would be+ # unverified — demote to REPLY.+ envelope = '{"type": "REPLACE_WORKFLOW", "user_response": "updated your workflow", "global_llm_context": {bad}}'+ parsed = parse_final_response(envelope)+ assert parsed["type"] == "REPLY"+ assert parsed["user_response"] == "updated your workflow"++ def test_envelope_shaped_unparseable_with_no_recoverable_user_response_returns_done(self) -> None:+ envelope = '{"type": "REPLY", "user_response": "broken'+ parsed = parse_final_response(envelope)+ assert parsed["user_response"] == "Done."+ assert parsed["type"] == "REPLY"+ assert "broken" not in parsed["user_response"]++ def test_non_envelope_unparseable_text_still_falls_back_to_text(self) -> None:+ text = "I'm not sure how to help with that."+ parsed = parse_final_response(text)+ assert parsed == {"type": "REPLY", "user_response": text}