Skip to content

Automate browser based workflows with AI

fix(sky-9287): copilot activity bullets — no raw JS / HTTP-header dumps (#5680)

Andrew Neilson0ba117ee5bf7
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
index 2fb194d5f..4eb8a1408 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
@@ -48,15 +48,21 @@ interface ToolActivity {
const TOOL_DISPLAY_NAMES: Record<string, string> = {
update_workflow: "Updating workflow",
+ update_and_run_blocks: "Updating & running blocks",
list_credentials: "Listing credentials",
get_block_schema: "Looking up block schema",
validate_block: "Validating block",
run_blocks_and_collect_debug: "Running blocks",
+ get_run_results: "Getting run results",
get_browser_screenshot: "Taking screenshot",
navigate_browser: "Navigating browser",
evaluate: "Evaluating JavaScript",
click: "Clicking element",
type_text: "Typing text",
+ scroll: "Scrolling",
+ console_messages: "Reading console",
+ select_option: "Selecting option",
+ press_key: "Pressing key",
};
const formatChatTimestamp = (value: string) => {
diff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.py
index 2a9d0ee7e..595e5b49e 100644
--- a/skyvern/forge/sdk/copilot/output_utils.py
+++ b/skyvern/forge/sdk/copilot/output_utils.py
@@ -5,6 +5,7 @@ from __future__ import annotations
import base64
import binascii
import json
+import re
from collections.abc import Iterator
from typing import TYPE_CHECKING, Any
@@ -182,10 +183,52 @@ def _extract_failure_message(result: dict[str, Any]) -> str:
return next(iter_failure_reasons(result), "Unknown error")
+_HEADERS_BLOB_RE = re.compile(r"\s*headers:\s*\{[^{}]*\}\s*", re.IGNORECASE)
+_LARGE_DICT_BLOB_RE = re.compile(r"\{[^{}]{40,}\}")
+
+
+def _sanitize_failure_text(text: str) -> str:
+ """Strip dict/HTTP-header dumps and cap a failure message for chat display.
+
+ The chat activity bullet is a fact, not a data dump — we never want raw
+ response headers or large JSON-looking blobs to flow into the SSE
+ payload. Short, capitalised technical tokens (``ERR_NAME_NOT_RESOLVED``)
+ must pass through unchanged."""
+ text = _HEADERS_BLOB_RE.sub(" ", text)
+ text = _LARGE_DICT_BLOB_RE.sub("{...}", text)
+ text = " ".join(text.split())
+ if not text:
+ return "(no details)"
+ if len(text) > 120:
+ text = text[:117] + "..."
+ return text
+
+
+def _describe_value_shape(value: Any) -> str:
+ """Describe the shape of a JS evaluation result without echoing values.
+
+ Distinct from ``_summarize_extracted_data``: that helper shapes data for
+ the LLM context (different verb, different audience). This one phrases
+ the shape for a chat activity bullet."""
+ if isinstance(value, list):
+ if not value:
+ return "empty list"
+ if isinstance(value[0], dict):
+ keys = sorted(value[0].keys())
+ return f"list of {len(value)} items, keys: {', '.join(keys)}"
+ return f"list of {len(value)} items"
+ if isinstance(value, dict):
+ keys = sorted(value.keys())
+ return f"object with keys: {', '.join(keys)}"
+ if isinstance(value, str):
+ return f"text ({len(value)} chars)"
+ return "value"
+
+
def summarize_tool_result(tool_name: str, result: dict[str, Any]) -> str:
"""Create a brief human-readable summary of a tool result."""
if not result.get("ok", False):
- return f"Failed: {_extract_failure_message(result)[:200]}"
+ return f"Failed: {_sanitize_failure_text(_extract_failure_message(result))}"
data = result.get("data") or {}
@@ -213,14 +256,16 @@ def summarize_tool_result(tool_name: str, result: dict[str, Any]) -> str:
return f"Run {', '.join(executed)}: {status}{suffix}"
return f"Run {', '.join(executed)}: {status}"
if tool_name == "get_browser_screenshot":
- return f"Screenshot taken ({data.get('url', '?')[:80]})"
+ url = data.get("url")
+ return f"Screenshot taken ({url[:80]})" if url else "Screenshot taken"
if tool_name == "navigate_browser":
url = result.get("url") or data.get("url", "?")
return f"Navigated to {url[:80]}"
if tool_name == "evaluate":
result_val = data.get("result")
- preview = str(result_val)[:100] if result_val is not None else "undefined"
- return f"JS result: {preview}"
+ if result_val is None:
+ return "Evaluated JavaScript"
+ return f"Evaluated JavaScript — returned {_describe_value_shape(result_val)}"
if tool_name == "click":
return f"Clicked '{data.get('selector', '?')}'"
if tool_name == "type_text":
diff --git a/tests/unit/test_copilot_output_utils.py b/tests/unit/test_copilot_output_utils.py
index c001b561c..60b548563 100644
--- a/tests/unit/test_copilot_output_utils.py
+++ b/tests/unit/test_copilot_output_utils.py
@@ -325,6 +325,83 @@ class TestSummarizeToolResult:
summary = self._summarize("unknown_tool", {"ok": True})
assert summary == "OK"
+ def test_evaluate_does_not_dump_raw_list(self) -> None:
+ # The activity bullet must describe shape only — JS return values
+ # (which are page-controlled) must never reach the SSE payload.
+ summary = self._summarize(
+ "evaluate",
+ {
+ "ok": True,
+ "data": {
+ "result": [
+ {"text": "Tickets", "href": "https://example.com/tickets/"},
+ {"text": "Hospitality", "href": "https://example.com/hospitality/"},
+ ]
+ },
+ },
+ )
+ assert "Tickets" not in summary
+ assert "Hospitality" not in summary
+ assert "example.com" not in summary
+ assert "list" in summary
+ assert "2" in summary
+
+ def test_evaluate_dict_returns_structural_summary(self) -> None:
+ summary = self._summarize(
+ "evaluate",
+ {
+ "ok": True,
+ "data": {"result": {"title": "Official Site", "url": "https://example.com/"}},
+ },
+ )
+ assert "Official Site" not in summary
+ assert "example.com" not in summary
+ assert "title" in summary # key names describe shape, not values
+ assert "url" in summary
+
+ def test_evaluate_none_returns_plain_label(self) -> None:
+ summary = self._summarize(
+ "evaluate",
+ {"ok": True, "data": {"result": None}},
+ )
+ assert summary == "Evaluated JavaScript"
+
+ def test_failure_strips_http_headers_blob(self) -> None:
+ # Failure summaries must never embed an HTTP-response-headers dict.
+ summary = self._summarize(
+ "click",
+ {
+ "ok": False,
+ "error": (
+ "headers: {'date': 'Mon, 27 Apr 2026 05:03:27 GMT', "
+ "'content-type': 'application/json', 'content-length': '43', "
+ "'connection': 'keep-alive'}"
+ ),
+ },
+ )
+ assert "'date'" not in summary
+ assert "'content-type'" not in summary
+ assert "keep-alive" not in summary
+ assert summary.startswith("Failed:")
+ assert len(summary) <= 128 # "Failed: " + ≤120 sanitized body
+
+ def test_failure_caps_at_120_chars(self) -> None:
+ long_message = "An unexpected error happened while doing the thing. " * 10
+ assert len(long_message) > 200
+ summary = self._summarize(
+ "click",
+ {"ok": False, "error": long_message},
+ )
+ body = summary[len("Failed: ") :]
+ assert len(body) <= 120
+
+ def test_screenshot_without_url_no_empty_parens(self) -> None:
+ summary = self._summarize(
+ "get_browser_screenshot",
+ {"ok": True, "data": {}},
+ )
+ assert summary == "Screenshot taken"
+
class TestParseFinalResponse:
"""parse_final_response is the last mile between model output and the frontend.