Skip to content

Automate browser based workflows with AI

feat(SKY-9353): distinguish self-healed copilot tool failures (#5732)

Andrew Neilson91511b4bf82d
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
index c4e7d02dc..70dad35dd 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
@@ -31,6 +31,12 @@ import {
WorkflowCopilotClearProposedWorkflowRequest,
WorkflowCopilotApplyProposedWorkflowRequest,
} from "./workflowCopilotTypes";
+import {
+ ToolActivity,
+ applyToolCall,
+ applyToolResult,
+ getActivityDotClass,
+} from "./toolActivity";
interface ChatMessage {
id: string;
@@ -48,13 +54,6 @@ type WorkflowCopilotSsePayload =
| WorkflowCopilotCondensingUpdate
| WorkflowCopilotNarrationUpdate;
-interface ToolActivity {
- tool_name: string;
- tool_call_id: string;
- status: "running" | "success" | "error";
- summary?: string;
-}
-
const TOOL_DISPLAY_NAMES: Record<string, string> = {
update_workflow: "Updating workflow",
update_and_run_blocks: "Updating & running blocks",
@@ -776,27 +775,10 @@ export function WorkflowCopilotChat({
TOOL_DISPLAY_NAMES[payload.tool_name] ??
payload.tool_name + "...",
);
- setToolActivity((prev) => [
- ...prev,
- {
- tool_name: payload.tool_name,
- tool_call_id: payload.tool_call_id,
- status: "running",
- },
- ]);
+ setToolActivity((prev) => applyToolCall(prev, payload));
return false;
case "tool_result":
- setToolActivity((prev) =>
- prev.map((item) =>
- item.tool_call_id === payload.tool_call_id
- ? {
- ...item,
- status: payload.success ? "success" : "error",
- summary: payload.summary,
- }
- : item,
- ),
- );
+ setToolActivity((prev) => applyToolResult(prev, payload));
return false;
case "condensing":
if (payload.status === "started") {
@@ -1181,27 +1163,38 @@ export function WorkflowCopilotChat({
</div>
{toolActivity.length > 0 && (
<div className="mt-2 space-y-1">
- {toolActivity.map((activity, index) => (
- <div
- key={index}
- className="flex items-start gap-1.5 text-xs text-slate-500"
- >
- <span
- className={`mt-1.5 inline-block h-1.5 w-1.5 shrink-0 rounded-full ${
- activity.status === "running"
- ? "animate-pulse bg-blue-400"
- : activity.status === "success"
- ? "bg-green-400"
- : "bg-red-400"
- }`}
- />
- <span className="line-clamp-2 min-w-0 flex-1">
- {TOOL_DISPLAY_NAMES[activity.tool_name] ??
- activity.tool_name}
- {activity.summary ? ` — ${activity.summary}` : ""}
- </span>
- </div>
- ))}
+ {toolActivity.map((activity) => {
+ const isRetrySuccess =
+ activity.status === "success" &&
+ activity.linkedRecovery === true;
+ const tooltip = activity.detail ?? activity.summary;
+ const displayName =
+ TOOL_DISPLAY_NAMES[activity.tool_name] ??
+ activity.tool_name;
+ const containerClass = `flex items-start gap-1.5 text-xs text-slate-500${
+ activity.linkedRecovery
+ ? " border-l border-amber-400/40 pl-2"
+ : ""
+ }`;
+ return (
+ <div
+ key={activity.tool_call_id}
+ className={containerClass}
+ >
+ <span
+ className={`mt-1.5 inline-block h-1.5 w-1.5 shrink-0 rounded-full ${getActivityDotClass(activity)}`}
+ />
+ <span
+ className="line-clamp-2 min-w-0 flex-1"
+ title={tooltip}
+ >
+ {isRetrySuccess ? "↻ " : ""}
+ {displayName}
+ {activity.summary ? ` — ${activity.summary}` : ""}
+ </span>
+ </div>
+ );
+ })}
</div>
)}
</div>
diff --git a/skyvern-frontend/src/routes/workflows/copilot/toolActivity.test.ts b/skyvern-frontend/src/routes/workflows/copilot/toolActivity.test.ts
new file mode 100644
index 000000000..fcaf0be88
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/copilot/toolActivity.test.ts
@@ -0,0 +1,337 @@
+import { describe, expect, it } from "vitest";
+
+import { ToolActivity, applyToolCall, applyToolResult } from "./toolActivity";
+import {
+ WorkflowCopilotToolCallUpdate,
+ WorkflowCopilotToolResultUpdate,
+} from "./workflowCopilotTypes";
+
+const make = (
+ overrides: Partial<ToolActivity> & Pick<ToolActivity, "tool_call_id">,
+): ToolActivity => ({
+ tool_name: "update_workflow",
+ status: "running",
+ ...overrides,
+});
+
+const result = (
+ overrides: Partial<WorkflowCopilotToolResultUpdate> &
+ Pick<WorkflowCopilotToolResultUpdate, "tool_call_id" | "success">,
+): WorkflowCopilotToolResultUpdate => ({
+ type: "tool_result",
+ tool_name: "update_workflow",
+ summary: "",
+ iteration: 0,
+ ...overrides,
+});
+
+const call = (
+ overrides: Partial<WorkflowCopilotToolCallUpdate> &
+ Pick<WorkflowCopilotToolCallUpdate, "tool_call_id">,
+): WorkflowCopilotToolCallUpdate => ({
+ type: "tool_call",
+ tool_name: "update_workflow",
+ tool_input: {},
+ iteration: 0,
+ ...overrides,
+});
+
+describe("applyToolResult", () => {
+ it("returns prev unchanged when tool_call_id has no matching entry", () => {
+ const prev = [make({ tool_call_id: "c1" })];
+ const next = applyToolResult(
+ prev,
+ result({ tool_call_id: "missing", success: true, summary: "ok" }),
+ );
+ expect(next).toBe(prev);
+ });
+
+ it("flips a running entry to error and copies detail", () => {
+ const prev = [make({ tool_call_id: "c1" })];
+ const next = applyToolResult(
+ prev,
+ result({
+ tool_call_id: "c1",
+ success: false,
+ summary: "Failed: short",
+ detail: "long sanitized error text for tooltip",
+ }),
+ );
+ expect(next[0]!).toMatchObject({
+ status: "error",
+ summary: "Failed: short",
+ detail: "long sanitized error text for tooltip",
+ });
+ });
+
+ it("normalizes payload null detail to undefined on the activity", () => {
+ const prev = [make({ tool_call_id: "c1" })];
+ const next = applyToolResult(
+ prev,
+ result({
+ tool_call_id: "c1",
+ success: false,
+ summary: "x",
+ detail: null,
+ }),
+ );
+ expect(next[0]!.detail).toBeUndefined();
+ });
+
+ it("links an adjacent failure to the next same-tool success", () => {
+ const prev: ToolActivity[] = [
+ make({
+ tool_name: "update_workflow",
+ tool_call_id: "c1",
+ status: "error",
+ summary: "Failed: bad yaml",
+ }),
+ make({
+ tool_name: "update_workflow",
+ tool_call_id: "c2",
+ status: "running",
+ }),
+ ];
+ const next = applyToolResult(
+ prev,
+ result({
+ tool_call_id: "c2",
+ success: true,
+ summary: "Workflow updated",
+ }),
+ );
+ expect(next[0]!).toMatchObject({
+ status: "error",
+ recovered: true,
+ linkedRecovery: true,
+ });
+ expect(next[1]!).toMatchObject({
+ status: "success",
+ linkedRecovery: true,
+ });
+ });
+
+ it("recovers but does NOT link a non-adjacent prior failure", () => {
+ const prev: ToolActivity[] = [
+ make({
+ tool_name: "update_workflow",
+ tool_call_id: "c1",
+ status: "error",
+ }),
+ make({
+ tool_name: "get_block_schema",
+ tool_call_id: "c2",
+ status: "success",
+ }),
+ make({
+ tool_name: "update_workflow",
+ tool_call_id: "c3",
+ status: "running",
+ }),
+ ];
+ const next = applyToolResult(
+ prev,
+ result({ tool_call_id: "c3", success: true, summary: "ok" }),
+ );
+ expect(next[0]!).toMatchObject({
+ status: "error",
+ recovered: true,
+ });
+ expect(next[0]!.linkedRecovery).toBeFalsy();
+ expect(next[2]!.linkedRecovery).toBeFalsy();
+ expect(next[2]!.status).toBe("success");
+ expect(next[1]!).toEqual(prev[1]!);
+ });
+
+ it("recovers ALL prior unrecovered same-tool failures, not just one", () => {
+ const prev: ToolActivity[] = [
+ make({
+ tool_name: "update_workflow",
+ tool_call_id: "c1",
+ status: "error",
+ }),
+ make({
+ tool_name: "update_workflow",
+ tool_call_id: "c2",
+ status: "error",
+ }),
+ make({
+ tool_name: "update_workflow",
+ tool_call_id: "c3",
+ status: "running",
+ }),
+ ];
+ const next = applyToolResult(
+ prev,
+ result({ tool_call_id: "c3", success: true, summary: "ok" }),
+ );
+ expect(next[0]!.recovered).toBe(true);
+ expect(next[1]!.recovered).toBe(true);
+ expect(next[0]!.linkedRecovery).toBeFalsy();
+ expect(next[1]!.linkedRecovery).toBe(true);
+ expect(next[2]!).toMatchObject({
+ linkedRecovery: true,
+ });
+ });
+
+ it("does not re-recover a failure that an earlier intermediate success already resolved", () => {
+ const prev: ToolActivity[] = [
+ {
+ tool_name: "update_workflow",
+ tool_call_id: "c1",
+ status: "error",
+ recovered: true,
+ linkedRecovery: true,
+ },
+ {
+ tool_name: "update_workflow",
+ tool_call_id: "c2",
+ status: "success",
+ linkedRecovery: true,
+ },
+ {
+ tool_name: "update_workflow",
+ tool_call_id: "c3",
+ status: "error",
+ },
+ {
+ tool_name: "update_workflow",
+ tool_call_id: "c4",
+ status: "running",
+ },
+ ];
+ const next = applyToolResult(
+ prev,
+ result({ tool_call_id: "c4", success: true, summary: "ok" }),
+ );
+ expect(next[0]).toBe(prev[0]);
+ expect(next[1]).toBe(prev[1]);
+ expect(next[2]!).toMatchObject({
+ recovered: true,
+ linkedRecovery: true,
+ });
+ expect(next[3]!).toMatchObject({
+ status: "success",
+ linkedRecovery: true,
+ });
+ });
+
+ it("links the adjacent retry success after applyToolCall has already marked the prior failure recovered", () => {
+ // Live flow: tool_call appends running and clears the pulse on the
+ // prior failure (recovered=true). Then tool_result success arrives and
+ // must still draw the connector on the adjacent pair.
+ let state: ToolActivity[] = [
+ {
+ tool_name: "update_workflow",
+ tool_call_id: "c1",
+ status: "error",
+ },
+ ];
+ state = applyToolCall(
+ state,
+ call({ tool_name: "update_workflow", tool_call_id: "c2" }),
+ );
+ expect(state[0]!.recovered).toBe(true);
+ expect(state[0]!.linkedRecovery).toBeUndefined();
+
+ state = applyToolResult(
+ state,
+ result({ tool_call_id: "c2", success: true, summary: "ok" }),
+ );
+ expect(state[0]!).toMatchObject({
+ status: "error",
+ recovered: true,
+ linkedRecovery: true,
+ });
+ expect(state[1]!).toMatchObject({
+ status: "success",
+ linkedRecovery: true,
+ });
+ });
+
+ it("does not link to a different tool's failure", () => {
+ const prev: ToolActivity[] = [
+ make({
+ tool_name: "click",
+ tool_call_id: "c1",
+ status: "error",
+ }),
+ make({
+ tool_name: "update_workflow",
+ tool_call_id: "c2",
+ status: "running",
+ }),
+ ];
+ const next = applyToolResult(
+ prev,
+ result({
+ tool_name: "update_workflow",
+ tool_call_id: "c2",
+ success: true,
+ summary: "ok",
+ }),
+ );
+ // Different tool: not linked, but the click failure stays unrecovered
+ // until a tool_call event clears it (handled by applyToolCall).
+ expect(next[0]!.recovered).toBeUndefined();
+ expect(next[1]!.linkedRecovery).toBeFalsy();
+ });
+});
+
+describe("applyToolCall", () => {
+ it("appends a running entry", () => {
+ const prev: ToolActivity[] = [];
+ const next = applyToolCall(
+ prev,
+ call({ tool_name: "update_workflow", tool_call_id: "c1" }),
+ );
+ expect(next).toHaveLength(1);
+ expect(next[0]!).toMatchObject({
+ tool_name: "update_workflow",
+ tool_call_id: "c1",
+ status: "running",
+ });
+ });
+
+ it("stops the pulse on prior unrecovered failures of any tool", () => {
+ const prev: ToolActivity[] = [
+ {
+ tool_name: "update_workflow",
+ tool_call_id: "c1",
+ status: "error",
+ },
+ {
+ tool_name: "evaluate",
+ tool_call_id: "c2",
+ status: "success",
+ },
+ ];
+ const next = applyToolCall(
+ prev,
+ call({ tool_name: "evaluate", tool_call_id: "c3" }),
+ );
+ expect(next[0]!.recovered).toBe(true);
+ expect(next[0]!.linkedRecovery).toBeUndefined();
+ expect(next[1]).toEqual(prev[1]);
+ expect(next[2]!).toMatchObject({
+ tool_call_id: "c3",
+ status: "running",
+ });
+ });
+
+ it("leaves already-recovered failures untouched", () => {
+ const prior: ToolActivity = {
+ tool_name: "update_workflow",
+ tool_call_id: "c1",
+ status: "error",
+ recovered: true,
+ linkedRecovery: true,
+ };
+ const prev: ToolActivity[] = [prior];
+ const next = applyToolCall(
+ prev,
+ call({ tool_name: "evaluate", tool_call_id: "c2" }),
+ );
+ expect(next[0]!).toBe(prior);
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/copilot/toolActivity.ts b/skyvern-frontend/src/routes/workflows/copilot/toolActivity.ts
new file mode 100644
index 000000000..f4bdc037f
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/copilot/toolActivity.ts
@@ -0,0 +1,126 @@
+import {
+ WorkflowCopilotToolCallUpdate,
+ WorkflowCopilotToolResultUpdate,
+} from "./workflowCopilotTypes";
+
+export interface ToolActivity {
+ tool_name: string;
+ tool_call_id: string;
+ status: "running" | "success" | "error";
+ summary?: string;
+ detail?: string;
+ recovered?: boolean;
+ // True for the adjacent failure/success pair only — gates the visual
+ // connector (left border + ↻). Non-adjacent retries get amber dot only.
+ linkedRecovery?: boolean;
+}
+
+export function getActivityDotClass(activity: ToolActivity): string {
+ if (activity.status === "running") {
+ return "animate-pulse bg-blue-400";
+ }
+ if (activity.status === "success") {
+ return "bg-green-400";
+ }
+ return activity.recovered ? "bg-amber-400" : "animate-pulse bg-amber-400";
+}
+
+// Append a new running entry and stop the pulse on any prior unrecovered
+// failure — once the agent has started anything new, the previous failure
+// is no longer the live event.
+export function applyToolCall(
+ prev: ToolActivity[],
+ payload: WorkflowCopilotToolCallUpdate,
+): ToolActivity[] {
+ const cleared = prev.map((item) =>
+ item.status === "error" && !item.recovered
+ ? { ...item, recovered: true }
+ : item,
+ );
+ return [
+ ...cleared,
+ {
+ tool_name: payload.tool_name,
+ tool_call_id: payload.tool_call_id,
+ status: "running" as const,
+ },
+ ];
+}
+
+// On success, marks every prior unrecovered same-tool error as `recovered`;
+// only the immediately adjacent failure also gets `linkedRecovery` so the
+// visual connector renders for that pair alone.
+export function applyToolResult(
+ prev: ToolActivity[],
+ payload: WorkflowCopilotToolResultUpdate,
+): ToolActivity[] {
+ const idx = prev.findIndex(
+ (item) => item.tool_call_id === payload.tool_call_id,
+ );
+ if (idx === -1) {
+ return prev;
+ }
+
+ // Wire shape allows null on optional fields; UI shape uses undefined.
+ const detail = payload.detail ?? undefined;
+
+ if (!payload.success) {
+ return prev.map((item, i) =>
+ i === idx
+ ? {
+ ...item,
+ status: "error" as const,
+ summary: payload.summary,
+ detail,
+ }
+ : item,
+ );
+ }
+
+ // Adjacency for the visual link is independent of `recovered` — by the
+ // time this fires, applyToolCall has already marked the prior failure
+ // recovered, but the connector still belongs on the immediately adjacent
+ // pair regardless.
+ const adjacent = prev[idx - 1];
+ const linkedFailureIdx =
+ adjacent &&
+ adjacent.status === "error" &&
+ adjacent.tool_name === payload.tool_name
+ ? idx - 1
+ : -1;
+
+ // Backwards scan covers the isolated case (applyToolResult invoked
+ // without applyToolCall having run first, e.g. unit tests, replay).
+ // In the live flow these entries are already recovered=true.
+ const recoveredIndices = new Set<number>();
+ for (let i = idx - 1; i >= 0; i -= 1) {
+ const candidate = prev[i];
+ if (
+ candidate &&
+ candidate.status === "error" &&
+ candidate.tool_name === payload.tool_name &&
+ !candidate.recovered
+ ) {
+ recoveredIndices.add(i);
+ }
+ }
+
+ return prev.map((item, i) => {
+ if (i === idx) {
+ return {
+ ...item,
+ status: "success" as const,
+ summary: payload.summary,
+ detail,
+ linkedRecovery: linkedFailureIdx !== -1,
+ };
+ }
+ if (i === linkedFailureIdx) {
+ return { ...item, recovered: true, linkedRecovery: true };
+ }
+ if (recoveredIndices.has(i)) {
+ return { ...item, recovered: true };
+ }
+ return item;
+ });
+}
diff --git a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
index c84a98831..dd2545926 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
@@ -104,6 +104,7 @@ export interface WorkflowCopilotToolResultUpdate {
summary: string;
iteration: number;
tool_call_id: string;
+ detail?: string | null;
}
export interface WorkflowCopilotCondensingUpdate {
diff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.py
index 5b90058a9..b26e06eef 100644
--- a/skyvern/forge/sdk/copilot/output_utils.py
+++ b/skyvern/forge/sdk/copilot/output_utils.py
@@ -247,7 +247,7 @@ _HEADERS_BLOB_RE = re.compile(r"\s*headers:\s*\{[^{}]*\}\s*", re.IGNORECASE)
_LARGE_DICT_BLOB_RE = re.compile(r"\{[^{}]{40,}\}")
-def _sanitize_failure_text(text: str) -> str:
+def _sanitize_failure_text(text: str, max_chars: int = 120) -> str:
"""Strip dict/HTTP-header dumps and cap a failure message for chat display.
The chat activity bullet is a fact, not a data dump — we never want raw
@@ -259,8 +259,8 @@ def _sanitize_failure_text(text: str) -> str:
text = " ".join(text.split())
if not text:
return "(no details)"
- if len(text) > 120:
- text = text[:117] + "..."
+ if len(text) > max_chars:
+ text = text[: max_chars - 3] + "..."
return text
@@ -343,6 +343,21 @@ def summarize_tool_result(tool_name: str, result: dict[str, Any]) -> str:
return "OK"
+def build_run_blocks_response(run_ok: bool, result_data: dict[str, Any]) -> dict[str, Any]:
+ """Wrap a run-blocks result, promoting the first failure reason to a top-level ``error``."""
+ response: dict[str, Any] = {"ok": run_ok, "data": result_data}
+ if not run_ok:
+ response["error"] = next(iter_failure_reasons(response), "Unknown error (no failure reason provided)")
+ return response
+
+
+def summarize_tool_result_detail(result: dict[str, Any], max_chars: int = 800) -> str | None:
+ """Tooltip-grade failure detail (longer cap than ``summarize_tool_result``); None on success."""
+ if result.get("ok", False):
+ return None
+ return _sanitize_failure_text(_extract_failure_message(result), max_chars=max_chars)
+
+
def truncate_output(output: Any, max_chars: int = 2000) -> str | None:
if output is None:
return None
diff --git a/skyvern/forge/sdk/copilot/streaming_adapter.py b/skyvern/forge/sdk/copilot/streaming_adapter.py
index 2417a32a7..ae1f58a09 100644
--- a/skyvern/forge/sdk/copilot/streaming_adapter.py
+++ b/skyvern/forge/sdk/copilot/streaming_adapter.py
@@ -21,7 +21,7 @@ from skyvern.forge.sdk.copilot.narration import (
schedule_narration,
snapshot_ctx,
)
-from skyvern.forge.sdk.copilot.output_utils import summarize_tool_result
+from skyvern.forge.sdk.copilot.output_utils import summarize_tool_result, summarize_tool_result_detail
from skyvern.forge.sdk.schemas.workflow_copilot import (
WorkflowCopilotStreamMessageType,
WorkflowCopilotToolCallUpdate,
@@ -147,6 +147,7 @@ async def stream_to_sse(
# below also needs them, and the work is cheap (no I/O).
summary = summarize_tool_result(tool_name, parsed)
success = parsed.get("ok", True)
+ detail = summarize_tool_result_detail(parsed)
if not client_gone:
await stream.send(
@@ -157,6 +158,7 @@ async def stream_to_sse(
summary=summary,
iteration=iteration,
tool_call_id=call_id,
+ detail=detail,
)
)
diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.py
index cd961310f..726ba37ab 100644
--- a/skyvern/forge/sdk/copilot/tools.py
+++ b/skyvern/forge/sdk/copilot/tools.py
@@ -31,6 +31,7 @@ from skyvern.forge.sdk.copilot.failure_tracking import (
from skyvern.forge.sdk.copilot.loop_detection import detect_tool_loop
from skyvern.forge.sdk.copilot.mcp_adapter import SchemaOverlay
from skyvern.forge.sdk.copilot.output_utils import (
+ build_run_blocks_response,
iter_failure_reasons,
sanitize_tool_result_for_llm,
truncate_output,
@@ -1371,15 +1372,7 @@ async def _run_blocks_and_collect_debug(
existing_set.add(label)
ctx.verified_prefix_labels = existing_prefix
- response: dict[str, Any] = {"ok": run_ok, "data": result_data}
- if not run_ok:
- # Promote the real failure message into a top-level ``error`` field so
- # every downstream consumer (summarize_tool_result, SSE frames, logs,
- # LLM context) sees it instead of defaulting to "Unknown error".
- # Wording matches ExtractionBlock's "Unknown error (no failure reason
- # provided)" fallback so error-message matching stays consistent.
- response["error"] = next(iter_failure_reasons(response), "Unknown error (no failure reason provided)")
- return response
+ return build_run_blocks_response(run_ok, result_data)
async def _get_run_results(params: dict[str, Any], ctx: AgentContext) -> dict[str, Any]:
diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.py
index e130655d5..036c22fe0 100644
--- a/skyvern/forge/sdk/schemas/workflow_copilot.py
+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py
@@ -142,6 +142,13 @@ class WorkflowCopilotToolResultUpdate(BaseModel):
summary: str = Field(..., description="Brief human-readable summary of the result")
iteration: int = Field(..., description="Agent loop iteration number")
tool_call_id: str = Field(..., description="Unique ID for this tool invocation")
+ detail: str | None = Field(
+ None,
+ description=(
+ "Longer-cap sanitized failure text for tooltip display. None on success. "
+ "Distinct from `summary`, which is capped tighter for the visible bullet."
+ ),
+ )
class WorkflowCopilotCondensingUpdate(BaseModel):
diff --git a/tests/unit/test_copilot_recoverable_failures.py b/tests/unit/test_copilot_recoverable_failures.py
new file mode 100644
index 000000000..60f5c7a12
--- /dev/null
+++ b/tests/unit/test_copilot_recoverable_failures.py
@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+import json
+from types import SimpleNamespace
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from agents.items import RunItem
+from agents.stream_events import RunItemStreamEvent
+
+from skyvern.forge.sdk.copilot.output_utils import (
+ _sanitize_failure_text,
+ build_run_blocks_response,
+ sanitize_tool_result_for_llm,
+ summarize_tool_result_detail,
+)
+from skyvern.forge.sdk.copilot.streaming_adapter import stream_to_sse
+from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotStreamMessageType
+
+
+def test_summarize_tool_result_detail_returns_none_on_success() -> None:
+ assert summarize_tool_result_detail({"ok": True, "data": {"block_count": 2}}) is None
+
+
+def test_summarize_tool_result_detail_caps_at_max_chars() -> None:
+ long_error = "Workflow validation failed: " + ("missing field 'foo'; " * 200)
+ detail = summarize_tool_result_detail({"ok": False, "error": long_error}, max_chars=400)
+ assert detail is not None
+ assert len(detail) <= 400
+ assert detail.endswith("...")
+
+
+def test_summarize_tool_result_detail_preserves_short_full_message() -> None:
+ detail = summarize_tool_result_detail(
+ {"ok": False, "error": "Workflow validation failed: title field required"},
+ )
+ assert detail == "Workflow validation failed: title field required"
+
+
+def test_summarize_tool_result_detail_strips_header_blobs() -> None:
+ text = "Failure with headers: {'host': 'x', 'authorization': 'Bearer abc'} please retry"
+ detail = summarize_tool_result_detail({"ok": False, "error": text})
+ assert detail is not None
+ assert "authorization" not in detail
+ assert "Bearer" not in detail
+
+
+def test_sanitize_failure_text_default_cap_unchanged() -> None:
+ sanitized = _sanitize_failure_text("x" * 200)
+ assert len(sanitized) == 120
+ assert sanitized.endswith("...")
+
+
+def test_sanitize_failure_text_respects_max_chars() -> None:
+ sanitized = _sanitize_failure_text("x" * 1000, max_chars=500)
+ assert len(sanitized) == 500
+ assert sanitized.endswith("...")
+
+
+def test_sanitize_tool_result_for_llm_passes_through_failure_dict() -> None:
+ failure = {"ok": False, "error": "Workflow validation failed: title required"}
+ sanitized = sanitize_tool_result_for_llm("update_workflow", failure)
+ assert sanitized["ok"] is False
+ assert sanitized["error"] == "Workflow validation failed: title required"
+
+
+def test_build_run_blocks_response_success_passes_through() -> None:
+ response = build_run_blocks_response(True, {"workflow_run_id": "wr_test", "blocks": []})
+ assert response == {"ok": True, "data": {"workflow_run_id": "wr_test", "blocks": []}}
+
+
+def test_build_run_blocks_response_promotes_run_level_failure_reason() -> None:
+ response = build_run_blocks_response(
+ False,
+ {
+ "workflow_run_id": "wr_test",
+ "overall_status": "failed",
+ "failure_reason": "Navigation timed out after 60s",
+ "blocks": [],
+ },
+ )
+ assert response["ok"] is False
+ assert response["error"] == "Navigation timed out after 60s"
+
+
+def test_build_run_blocks_response_falls_back_when_no_failure_reason() -> None:
+ response = build_run_blocks_response(False, {"workflow_run_id": "wr_test"})
+ assert response["error"] == "Unknown error (no failure reason provided)"
+
+
+async def _capture_tool_result(tool_name: str, parsed_output: dict[str, Any]) -> Any:
+ """Drive `stream_to_sse` over a single tool round-trip and return the
+ emitted ``WorkflowCopilotToolResultUpdate``."""
+ call_item = MagicMock(spec=RunItem)
+ call_item.raw_item = {"call_id": "c1", "name": tool_name, "arguments": "{}"}
+ tool_call = RunItemStreamEvent(name="tool_called", item=call_item)
+
+ out_item = MagicMock(spec=RunItem)
+ out_item.raw_item = {"call_id": "c1", "name": tool_name}
+ out_item.output = [{"type": "text", "text": json.dumps(parsed_output)}]
+ tool_output = RunItemStreamEvent(name="tool_output", item=out_item)
+
+ async def _events() -> Any:
+ yield tool_call
+ yield tool_output
+
+ result = MagicMock()
+ result.stream_events = lambda: _events()
+ result.cancel = MagicMock()
+
+ sent: list[Any] = []
+
+ async def _send(payload: Any) -> bool:
+ sent.append(payload)
+ return True
+
+ stream = MagicMock()
+ stream.is_disconnected = AsyncMock(return_value=False)
+ stream.send = _send
+
+ await stream_to_sse(result, stream, SimpleNamespace())
+
+ tool_results = [p for p in sent if getattr(p, "type", None) == WorkflowCopilotStreamMessageType.TOOL_RESULT]
+ assert len(tool_results) == 1
+ return tool_results[0]
+
+
+async def test_stream_emits_detail_for_failure() -> None:
+ long_error = "Workflow validation failed: " + (
+ "blocks.0.task expects 'navigation_goal' but the emitted YAML omitted it. " * 4
+ )
+ payload = await _capture_tool_result(
+ "update_workflow",
+ {"ok": False, "error": long_error},
+ )
+ assert payload.success is False
+ assert payload.detail is not None
+ assert len(payload.detail) > 120
+ # `summary` is the visible bullet, capped tighter than `detail` (the
+ # tooltip-grade text) — strictly longer detail is the contract.
+ assert len(payload.detail) > len(payload.summary)
+
+
+async def test_stream_emits_no_detail_for_success() -> None:
+ payload = await _capture_tool_result(
+ "update_workflow",
+ {"ok": True, "data": {"block_count": 3}},
+ )
+ assert payload.success is True
+ assert payload.detail is None