Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(sky-9287): copilot activity bullets — no raw JS / HTTP-header dumps (#5680)
Andrew Neilson0ba117ee5bf7
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsxindex 2fb194d5f..4eb8a1408 100644--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx@@ -48,15 +48,21 @@ interface ToolActivity {const TOOL_DISPLAY_NAMES: Record<string, string> = {update_workflow: "Updating workflow",+ update_and_run_blocks: "Updating & running blocks",list_credentials: "Listing credentials",get_block_schema: "Looking up block schema",validate_block: "Validating block",run_blocks_and_collect_debug: "Running blocks",+ get_run_results: "Getting run results",get_browser_screenshot: "Taking screenshot",navigate_browser: "Navigating browser",evaluate: "Evaluating JavaScript",click: "Clicking element",type_text: "Typing text",+ scroll: "Scrolling",+ console_messages: "Reading console",+ select_option: "Selecting option",+ press_key: "Pressing key",};const formatChatTimestamp = (value: string) => {diff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.pyindex 2a9d0ee7e..595e5b49e 100644--- a/skyvern/forge/sdk/copilot/output_utils.py+++ b/skyvern/forge/sdk/copilot/output_utils.py@@ -5,6 +5,7 @@ from __future__ import annotationsimport base64import binasciiimport json+import refrom collections.abc import Iteratorfrom typing import TYPE_CHECKING, Any@@ -182,10 +183,52 @@ def _extract_failure_message(result: dict[str, Any]) -> str:return next(iter_failure_reasons(result), "Unknown error")+_HEADERS_BLOB_RE = re.compile(r"\s*headers:\s*\{[^{}]*\}\s*", re.IGNORECASE)+_LARGE_DICT_BLOB_RE = re.compile(r"\{[^{}]{40,}\}")+++def _sanitize_failure_text(text: str) -> str:+ """Strip dict/HTTP-header dumps and cap a failure message for chat display.++ The chat activity bullet is a fact, not a data dump — we never want raw+ response headers or large JSON-looking blobs to flow into the SSE+ payload. Short, capitalised technical tokens (``ERR_NAME_NOT_RESOLVED``)+ must pass through unchanged."""+ text = _HEADERS_BLOB_RE.sub(" ", text)+ text = _LARGE_DICT_BLOB_RE.sub("{...}", text)+ text = " ".join(text.split())+ if not text:+ return "(no details)"+ if len(text) > 120:+ text = text[:117] + "..."+ return text+++def _describe_value_shape(value: Any) -> str:+ """Describe the shape of a JS evaluation result without echoing values.++ Distinct from ``_summarize_extracted_data``: that helper shapes data for+ the LLM context (different verb, different audience). This one phrases+ the shape for a chat activity bullet."""+ if isinstance(value, list):+ if not value:+ return "empty list"+ if isinstance(value[0], dict):+ keys = sorted(value[0].keys())+ return f"list of {len(value)} items, keys: {', '.join(keys)}"+ return f"list of {len(value)} items"+ if isinstance(value, dict):+ keys = sorted(value.keys())+ return f"object with keys: {', '.join(keys)}"+ if isinstance(value, str):+ return f"text ({len(value)} chars)"+ return "value"++def summarize_tool_result(tool_name: str, result: dict[str, Any]) -> str:"""Create a brief human-readable summary of a tool result."""if not result.get("ok", False):- return f"Failed: {_extract_failure_message(result)[:200]}"+ return f"Failed: {_sanitize_failure_text(_extract_failure_message(result))}"data = result.get("data") or {}@@ -213,14 +256,16 @@ def summarize_tool_result(tool_name: str, result: dict[str, Any]) -> str:return f"Run {', '.join(executed)}: {status}{suffix}"return f"Run {', '.join(executed)}: {status}"if tool_name == "get_browser_screenshot":- return f"Screenshot taken ({data.get('url', '?')[:80]})"+ url = data.get("url")+ return f"Screenshot taken ({url[:80]})" if url else "Screenshot taken"if tool_name == "navigate_browser":url = result.get("url") or data.get("url", "?")return f"Navigated to {url[:80]}"if tool_name == "evaluate":result_val = data.get("result")- preview = str(result_val)[:100] if result_val is not None else "undefined"- return f"JS result: {preview}"+ if result_val is None:+ return "Evaluated JavaScript"+ return f"Evaluated JavaScript — returned {_describe_value_shape(result_val)}"if tool_name == "click":return f"Clicked '{data.get('selector', '?')}'"if tool_name == "type_text":diff --git a/tests/unit/test_copilot_output_utils.py b/tests/unit/test_copilot_output_utils.pyindex c001b561c..60b548563 100644--- a/tests/unit/test_copilot_output_utils.py+++ b/tests/unit/test_copilot_output_utils.py@@ -325,6 +325,83 @@ class TestSummarizeToolResult:summary = self._summarize("unknown_tool", {"ok": True})assert summary == "OK"+ def test_evaluate_does_not_dump_raw_list(self) -> None:+ # The activity bullet must describe shape only — JS return values+ # (which are page-controlled) must never reach the SSE payload.+ summary = self._summarize(+ "evaluate",+ {+ "ok": True,+ "data": {+ "result": [+ {"text": "Tickets", "href": "https://example.com/tickets/"},+ {"text": "Hospitality", "href": "https://example.com/hospitality/"},+ ]+ },+ },+ )+ assert "Tickets" not in summary+ assert "Hospitality" not in summary+ assert "example.com" not in summary+ assert "list" in summary+ assert "2" in summary++ def test_evaluate_dict_returns_structural_summary(self) -> None:+ summary = self._summarize(+ "evaluate",+ {+ "ok": True,+ "data": {"result": {"title": "Official Site", "url": "https://example.com/"}},+ },+ )+ assert "Official Site" not in summary+ assert "example.com" not in summary+ assert "title" in summary # key names describe shape, not values+ assert "url" in summary++ def test_evaluate_none_returns_plain_label(self) -> None:+ summary = self._summarize(+ "evaluate",+ {"ok": True, "data": {"result": None}},+ )+ assert summary == "Evaluated JavaScript"++ def test_failure_strips_http_headers_blob(self) -> None:+ # Failure summaries must never embed an HTTP-response-headers dict.+ summary = self._summarize(+ "click",+ {+ "ok": False,+ "error": (+ "headers: {'date': 'Mon, 27 Apr 2026 05:03:27 GMT', "+ "'content-type': 'application/json', 'content-length': '43', "+ "'connection': 'keep-alive'}"+ ),+ },+ )+ assert "'date'" not in summary+ assert "'content-type'" not in summary+ assert "keep-alive" not in summary+ assert summary.startswith("Failed:")+ assert len(summary) <= 128 # "Failed: " + ≤120 sanitized body++ def test_failure_caps_at_120_chars(self) -> None:+ long_message = "An unexpected error happened while doing the thing. " * 10+ assert len(long_message) > 200+ summary = self._summarize(+ "click",+ {"ok": False, "error": long_message},+ )+ body = summary[len("Failed: ") :]+ assert len(body) <= 120++ def test_screenshot_without_url_no_empty_parens(self) -> None:+ summary = self._summarize(+ "get_browser_screenshot",+ {"ok": True, "data": {}},+ )+ assert summary == "Screenshot taken"+class TestParseFinalResponse:"""parse_final_response is the last mile between model output and the frontend.