Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(SKY-9353): distinguish self-healed copilot tool failures (#5732)
Andrew Neilson91511b4bf82d
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsxindex c4e7d02dc..70dad35dd 100644--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx@@ -31,6 +31,12 @@ import {WorkflowCopilotClearProposedWorkflowRequest,WorkflowCopilotApplyProposedWorkflowRequest,} from "./workflowCopilotTypes";+import {+ ToolActivity,+ applyToolCall,+ applyToolResult,+ getActivityDotClass,+} from "./toolActivity";interface ChatMessage {id: string;@@ -48,13 +54,6 @@ type WorkflowCopilotSsePayload =| WorkflowCopilotCondensingUpdate| WorkflowCopilotNarrationUpdate;-interface ToolActivity {- tool_name: string;- tool_call_id: string;- status: "running" | "success" | "error";- summary?: string;-}-const TOOL_DISPLAY_NAMES: Record<string, string> = {update_workflow: "Updating workflow",update_and_run_blocks: "Updating & running blocks",@@ -776,27 +775,10 @@ export function WorkflowCopilotChat({TOOL_DISPLAY_NAMES[payload.tool_name] ??payload.tool_name + "...",);- setToolActivity((prev) => [- ...prev,- {- tool_name: payload.tool_name,- tool_call_id: payload.tool_call_id,- status: "running",- },- ]);+ setToolActivity((prev) => applyToolCall(prev, payload));return false;case "tool_result":- setToolActivity((prev) =>- prev.map((item) =>- item.tool_call_id === payload.tool_call_id- ? {- ...item,- status: payload.success ? "success" : "error",- summary: payload.summary,- }- : item,- ),- );+ setToolActivity((prev) => applyToolResult(prev, payload));return false;case "condensing":if (payload.status === "started") {@@ -1181,27 +1163,38 @@ export function WorkflowCopilotChat({</div>{toolActivity.length > 0 && (<div className="mt-2 space-y-1">- {toolActivity.map((activity, index) => (- <div- key={index}- className="flex items-start gap-1.5 text-xs text-slate-500"- >- <span- className={`mt-1.5 inline-block h-1.5 w-1.5 shrink-0 rounded-full ${- activity.status === "running"- ? "animate-pulse bg-blue-400"- : activity.status === "success"- ? "bg-green-400"- : "bg-red-400"- }`}- />- <span className="line-clamp-2 min-w-0 flex-1">- {TOOL_DISPLAY_NAMES[activity.tool_name] ??- activity.tool_name}- {activity.summary ? ` — ${activity.summary}` : ""}- </span>- </div>- ))}+ {toolActivity.map((activity) => {+ const isRetrySuccess =+ activity.status === "success" &&+ activity.linkedRecovery === true;+ const tooltip = activity.detail ?? activity.summary;+ const displayName =+ TOOL_DISPLAY_NAMES[activity.tool_name] ??+ activity.tool_name;+ const containerClass = `flex items-start gap-1.5 text-xs text-slate-500${+ activity.linkedRecovery+ ? " border-l border-amber-400/40 pl-2"+ : ""+ }`;+ return (+ <div+ key={activity.tool_call_id}+ className={containerClass}+ >+ <span+ className={`mt-1.5 inline-block h-1.5 w-1.5 shrink-0 rounded-full ${getActivityDotClass(activity)}`}+ />+ <span+ className="line-clamp-2 min-w-0 flex-1"+ title={tooltip}+ >+ {isRetrySuccess ? "↻ " : ""}+ {displayName}+ {activity.summary ? ` — ${activity.summary}` : ""}+ </span>+ </div>+ );+ })}</div>)}</div>diff --git a/skyvern-frontend/src/routes/workflows/copilot/toolActivity.test.ts b/skyvern-frontend/src/routes/workflows/copilot/toolActivity.test.tsnew file mode 100644index 000000000..fcaf0be88--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/copilot/toolActivity.test.ts@@ -0,0 +1,337 @@+import { describe, expect, it } from "vitest";++import { ToolActivity, applyToolCall, applyToolResult } from "./toolActivity";+import {+ WorkflowCopilotToolCallUpdate,+ WorkflowCopilotToolResultUpdate,+} from "./workflowCopilotTypes";++const make = (+ overrides: Partial<ToolActivity> & Pick<ToolActivity, "tool_call_id">,+): ToolActivity => ({+ tool_name: "update_workflow",+ status: "running",+ ...overrides,+});++const result = (+ overrides: Partial<WorkflowCopilotToolResultUpdate> &+ Pick<WorkflowCopilotToolResultUpdate, "tool_call_id" | "success">,+): WorkflowCopilotToolResultUpdate => ({+ type: "tool_result",+ tool_name: "update_workflow",+ summary: "",+ iteration: 0,+ ...overrides,+});++const call = (+ overrides: Partial<WorkflowCopilotToolCallUpdate> &+ Pick<WorkflowCopilotToolCallUpdate, "tool_call_id">,+): WorkflowCopilotToolCallUpdate => ({+ type: "tool_call",+ tool_name: "update_workflow",+ tool_input: {},+ iteration: 0,+ ...overrides,+});++describe("applyToolResult", () => {+ it("returns prev unchanged when tool_call_id has no matching entry", () => {+ const prev = [make({ tool_call_id: "c1" })];+ const next = applyToolResult(+ prev,+ result({ tool_call_id: "missing", success: true, summary: "ok" }),+ );+ expect(next).toBe(prev);+ });++ it("flips a running entry to error and copies detail", () => {+ const prev = [make({ tool_call_id: "c1" })];+ const next = applyToolResult(+ prev,+ result({+ tool_call_id: "c1",+ success: false,+ summary: "Failed: short",+ detail: "long sanitized error text for tooltip",+ }),+ );+ expect(next[0]!).toMatchObject({+ status: "error",+ summary: "Failed: short",+ detail: "long sanitized error text for tooltip",+ });+ });++ it("normalizes payload null detail to undefined on the activity", () => {+ const prev = [make({ tool_call_id: "c1" })];+ const next = applyToolResult(+ prev,+ result({+ tool_call_id: "c1",+ success: false,+ summary: "x",+ detail: null,+ }),+ );+ expect(next[0]!.detail).toBeUndefined();+ });++ it("links an adjacent failure to the next same-tool success", () => {+ const prev: ToolActivity[] = [+ make({+ tool_name: "update_workflow",+ tool_call_id: "c1",+ status: "error",+ summary: "Failed: bad yaml",+ }),+ make({+ tool_name: "update_workflow",+ tool_call_id: "c2",+ status: "running",+ }),+ ];+ const next = applyToolResult(+ prev,+ result({+ tool_call_id: "c2",+ success: true,+ summary: "Workflow updated",+ }),+ );+ expect(next[0]!).toMatchObject({+ status: "error",+ recovered: true,+ linkedRecovery: true,+ });+ expect(next[1]!).toMatchObject({+ status: "success",+ linkedRecovery: true,+ });+ });++ it("recovers but does NOT link a non-adjacent prior failure", () => {+ const prev: ToolActivity[] = [+ make({+ tool_name: "update_workflow",+ tool_call_id: "c1",+ status: "error",+ }),+ make({+ tool_name: "get_block_schema",+ tool_call_id: "c2",+ status: "success",+ }),+ make({+ tool_name: "update_workflow",+ tool_call_id: "c3",+ status: "running",+ }),+ ];+ const next = applyToolResult(+ prev,+ result({ tool_call_id: "c3", success: true, summary: "ok" }),+ );+ expect(next[0]!).toMatchObject({+ status: "error",+ recovered: true,+ });+ expect(next[0]!.linkedRecovery).toBeFalsy();+ expect(next[2]!.linkedRecovery).toBeFalsy();+ expect(next[2]!.status).toBe("success");+ expect(next[1]!).toEqual(prev[1]!);+ });++ it("recovers ALL prior unrecovered same-tool failures, not just one", () => {+ const prev: ToolActivity[] = [+ make({+ tool_name: "update_workflow",+ tool_call_id: "c1",+ status: "error",+ }),+ make({+ tool_name: "update_workflow",+ tool_call_id: "c2",+ status: "error",+ }),+ make({+ tool_name: "update_workflow",+ tool_call_id: "c3",+ status: "running",+ }),+ ];+ const next = applyToolResult(+ prev,+ result({ tool_call_id: "c3", success: true, summary: "ok" }),+ );+ expect(next[0]!.recovered).toBe(true);+ expect(next[1]!.recovered).toBe(true);+ expect(next[0]!.linkedRecovery).toBeFalsy();+ expect(next[1]!.linkedRecovery).toBe(true);+ expect(next[2]!).toMatchObject({+ linkedRecovery: true,+ });+ });++ it("does not re-recover a failure that an earlier intermediate success already resolved", () => {+ const prev: ToolActivity[] = [+ {+ tool_name: "update_workflow",+ tool_call_id: "c1",+ status: "error",+ recovered: true,+ linkedRecovery: true,+ },+ {+ tool_name: "update_workflow",+ tool_call_id: "c2",+ status: "success",+ linkedRecovery: true,+ },+ {+ tool_name: "update_workflow",+ tool_call_id: "c3",+ status: "error",+ },+ {+ tool_name: "update_workflow",+ tool_call_id: "c4",+ status: "running",+ },+ ];+ const next = applyToolResult(+ prev,+ result({ tool_call_id: "c4", success: true, summary: "ok" }),+ );+ expect(next[0]).toBe(prev[0]);+ expect(next[1]).toBe(prev[1]);+ expect(next[2]!).toMatchObject({+ recovered: true,+ linkedRecovery: true,+ });+ expect(next[3]!).toMatchObject({+ status: "success",+ linkedRecovery: true,+ });+ });++ it("links the adjacent retry success after applyToolCall has already marked the prior failure recovered", () => {+ // Live flow: tool_call appends running and clears the pulse on the+ // prior failure (recovered=true). Then tool_result success arrives and+ // must still draw the connector on the adjacent pair.+ let state: ToolActivity[] = [+ {+ tool_name: "update_workflow",+ tool_call_id: "c1",+ status: "error",+ },+ ];+ state = applyToolCall(+ state,+ call({ tool_name: "update_workflow", tool_call_id: "c2" }),+ );+ expect(state[0]!.recovered).toBe(true);+ expect(state[0]!.linkedRecovery).toBeUndefined();++ state = applyToolResult(+ state,+ result({ tool_call_id: "c2", success: true, summary: "ok" }),+ );+ expect(state[0]!).toMatchObject({+ status: "error",+ recovered: true,+ linkedRecovery: true,+ });+ expect(state[1]!).toMatchObject({+ status: "success",+ linkedRecovery: true,+ });+ });++ it("does not link to a different tool's failure", () => {+ const prev: ToolActivity[] = [+ make({+ tool_name: "click",+ tool_call_id: "c1",+ status: "error",+ }),+ make({+ tool_name: "update_workflow",+ tool_call_id: "c2",+ status: "running",+ }),+ ];+ const next = applyToolResult(+ prev,+ result({+ tool_name: "update_workflow",+ tool_call_id: "c2",+ success: true,+ summary: "ok",+ }),+ );+ // Different tool: not linked, but the click failure stays unrecovered+ // until a tool_call event clears it (handled by applyToolCall).+ expect(next[0]!.recovered).toBeUndefined();+ expect(next[1]!.linkedRecovery).toBeFalsy();+ });+});++describe("applyToolCall", () => {+ it("appends a running entry", () => {+ const prev: ToolActivity[] = [];+ const next = applyToolCall(+ prev,+ call({ tool_name: "update_workflow", tool_call_id: "c1" }),+ );+ expect(next).toHaveLength(1);+ expect(next[0]!).toMatchObject({+ tool_name: "update_workflow",+ tool_call_id: "c1",+ status: "running",+ });+ });++ it("stops the pulse on prior unrecovered failures of any tool", () => {+ const prev: ToolActivity[] = [+ {+ tool_name: "update_workflow",+ tool_call_id: "c1",+ status: "error",+ },+ {+ tool_name: "evaluate",+ tool_call_id: "c2",+ status: "success",+ },+ ];+ const next = applyToolCall(+ prev,+ call({ tool_name: "evaluate", tool_call_id: "c3" }),+ );+ expect(next[0]!.recovered).toBe(true);+ expect(next[0]!.linkedRecovery).toBeUndefined();+ expect(next[1]).toEqual(prev[1]);+ expect(next[2]!).toMatchObject({+ tool_call_id: "c3",+ status: "running",+ });+ });++ it("leaves already-recovered failures untouched", () => {+ const prior: ToolActivity = {+ tool_name: "update_workflow",+ tool_call_id: "c1",+ status: "error",+ recovered: true,+ linkedRecovery: true,+ };+ const prev: ToolActivity[] = [prior];+ const next = applyToolCall(+ prev,+ call({ tool_name: "evaluate", tool_call_id: "c2" }),+ );+ expect(next[0]!).toBe(prior);+ });+});diff --git a/skyvern-frontend/src/routes/workflows/copilot/toolActivity.ts b/skyvern-frontend/src/routes/workflows/copilot/toolActivity.tsnew file mode 100644index 000000000..f4bdc037f--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/copilot/toolActivity.ts@@ -0,0 +1,126 @@+import {+ WorkflowCopilotToolCallUpdate,+ WorkflowCopilotToolResultUpdate,+} from "./workflowCopilotTypes";++export interface ToolActivity {+ tool_name: string;+ tool_call_id: string;+ status: "running" | "success" | "error";+ summary?: string;+ detail?: string;+ recovered?: boolean;+ // True for the adjacent failure/success pair only — gates the visual+ // connector (left border + ↻). Non-adjacent retries get amber dot only.+ linkedRecovery?: boolean;+}++export function getActivityDotClass(activity: ToolActivity): string {+ if (activity.status === "running") {+ return "animate-pulse bg-blue-400";+ }+ if (activity.status === "success") {+ return "bg-green-400";+ }+ return activity.recovered ? "bg-amber-400" : "animate-pulse bg-amber-400";+}++// Append a new running entry and stop the pulse on any prior unrecovered+// failure — once the agent has started anything new, the previous failure+// is no longer the live event.+export function applyToolCall(+ prev: ToolActivity[],+ payload: WorkflowCopilotToolCallUpdate,+): ToolActivity[] {+ const cleared = prev.map((item) =>+ item.status === "error" && !item.recovered+ ? { ...item, recovered: true }+ : item,+ );+ return [+ ...cleared,+ {+ tool_name: payload.tool_name,+ tool_call_id: payload.tool_call_id,+ status: "running" as const,+ },+ ];+}++// On success, marks every prior unrecovered same-tool error as `recovered`;+// only the immediately adjacent failure also gets `linkedRecovery` so the+// visual connector renders for that pair alone.+export function applyToolResult(+ prev: ToolActivity[],+ payload: WorkflowCopilotToolResultUpdate,+): ToolActivity[] {+ const idx = prev.findIndex(+ (item) => item.tool_call_id === payload.tool_call_id,+ );+ if (idx === -1) {+ return prev;+ }++ // Wire shape allows null on optional fields; UI shape uses undefined.+ const detail = payload.detail ?? undefined;++ if (!payload.success) {+ return prev.map((item, i) =>+ i === idx+ ? {+ ...item,+ status: "error" as const,+ summary: payload.summary,+ detail,+ }+ : item,+ );+ }++ // Adjacency for the visual link is independent of `recovered` — by the+ // time this fires, applyToolCall has already marked the prior failure+ // recovered, but the connector still belongs on the immediately adjacent+ // pair regardless.+ const adjacent = prev[idx - 1];+ const linkedFailureIdx =+ adjacent &&+ adjacent.status === "error" &&+ adjacent.tool_name === payload.tool_name+ ? idx - 1+ : -1;++ // Backwards scan covers the isolated case (applyToolResult invoked+ // without applyToolCall having run first, e.g. unit tests, replay).+ // In the live flow these entries are already recovered=true.+ const recoveredIndices = new Set<number>();+ for (let i = idx - 1; i >= 0; i -= 1) {+ const candidate = prev[i];+ if (+ candidate &&+ candidate.status === "error" &&+ candidate.tool_name === payload.tool_name &&+ !candidate.recovered+ ) {+ recoveredIndices.add(i);+ }+ }++ return prev.map((item, i) => {+ if (i === idx) {+ return {+ ...item,+ status: "success" as const,+ summary: payload.summary,+ detail,+ linkedRecovery: linkedFailureIdx !== -1,+ };+ }+ if (i === linkedFailureIdx) {+ return { ...item, recovered: true, linkedRecovery: true };+ }+ if (recoveredIndices.has(i)) {+ return { ...item, recovered: true };+ }+ return item;+ });+}diff --git a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.tsindex c84a98831..dd2545926 100644--- a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts@@ -104,6 +104,7 @@ export interface WorkflowCopilotToolResultUpdate {summary: string;iteration: number;tool_call_id: string;+ detail?: string | null;}export interface WorkflowCopilotCondensingUpdate {diff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.pyindex 5b90058a9..b26e06eef 100644--- a/skyvern/forge/sdk/copilot/output_utils.py+++ b/skyvern/forge/sdk/copilot/output_utils.py@@ -247,7 +247,7 @@ _HEADERS_BLOB_RE = re.compile(r"\s*headers:\s*\{[^{}]*\}\s*", re.IGNORECASE)_LARGE_DICT_BLOB_RE = re.compile(r"\{[^{}]{40,}\}")-def _sanitize_failure_text(text: str) -> str:+def _sanitize_failure_text(text: str, max_chars: int = 120) -> str:"""Strip dict/HTTP-header dumps and cap a failure message for chat display.The chat activity bullet is a fact, not a data dump — we never want raw@@ -259,8 +259,8 @@ def _sanitize_failure_text(text: str) -> str:text = " ".join(text.split())if not text:return "(no details)"- if len(text) > 120:- text = text[:117] + "..."+ if len(text) > max_chars:+ text = text[: max_chars - 3] + "..."return text@@ -343,6 +343,21 @@ def summarize_tool_result(tool_name: str, result: dict[str, Any]) -> str:return "OK"+def build_run_blocks_response(run_ok: bool, result_data: dict[str, Any]) -> dict[str, Any]:+ """Wrap a run-blocks result, promoting the first failure reason to a top-level ``error``."""+ response: dict[str, Any] = {"ok": run_ok, "data": result_data}+ if not run_ok:+ response["error"] = next(iter_failure_reasons(response), "Unknown error (no failure reason provided)")+ return response+++def summarize_tool_result_detail(result: dict[str, Any], max_chars: int = 800) -> str | None:+ """Tooltip-grade failure detail (longer cap than ``summarize_tool_result``); None on success."""+ if result.get("ok", False):+ return None+ return _sanitize_failure_text(_extract_failure_message(result), max_chars=max_chars)++def truncate_output(output: Any, max_chars: int = 2000) -> str | None:if output is None:return Nonediff --git a/skyvern/forge/sdk/copilot/streaming_adapter.py b/skyvern/forge/sdk/copilot/streaming_adapter.pyindex 2417a32a7..ae1f58a09 100644--- a/skyvern/forge/sdk/copilot/streaming_adapter.py+++ b/skyvern/forge/sdk/copilot/streaming_adapter.py@@ -21,7 +21,7 @@ from skyvern.forge.sdk.copilot.narration import (schedule_narration,snapshot_ctx,)-from skyvern.forge.sdk.copilot.output_utils import summarize_tool_result+from skyvern.forge.sdk.copilot.output_utils import summarize_tool_result, summarize_tool_result_detailfrom skyvern.forge.sdk.schemas.workflow_copilot import (WorkflowCopilotStreamMessageType,WorkflowCopilotToolCallUpdate,@@ -147,6 +147,7 @@ async def stream_to_sse(# below also needs them, and the work is cheap (no I/O).summary = summarize_tool_result(tool_name, parsed)success = parsed.get("ok", True)+ detail = summarize_tool_result_detail(parsed)if not client_gone:await stream.send(@@ -157,6 +158,7 @@ async def stream_to_sse(summary=summary,iteration=iteration,tool_call_id=call_id,+ detail=detail,))diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.pyindex cd961310f..726ba37ab 100644--- a/skyvern/forge/sdk/copilot/tools.py+++ b/skyvern/forge/sdk/copilot/tools.py@@ -31,6 +31,7 @@ from skyvern.forge.sdk.copilot.failure_tracking import (from skyvern.forge.sdk.copilot.loop_detection import detect_tool_loopfrom skyvern.forge.sdk.copilot.mcp_adapter import SchemaOverlayfrom skyvern.forge.sdk.copilot.output_utils import (+ build_run_blocks_response,iter_failure_reasons,sanitize_tool_result_for_llm,truncate_output,@@ -1371,15 +1372,7 @@ async def _run_blocks_and_collect_debug(existing_set.add(label)ctx.verified_prefix_labels = existing_prefix- response: dict[str, Any] = {"ok": run_ok, "data": result_data}- if not run_ok:- # Promote the real failure message into a top-level ``error`` field so- # every downstream consumer (summarize_tool_result, SSE frames, logs,- # LLM context) sees it instead of defaulting to "Unknown error".- # Wording matches ExtractionBlock's "Unknown error (no failure reason- # provided)" fallback so error-message matching stays consistent.- response["error"] = next(iter_failure_reasons(response), "Unknown error (no failure reason provided)")- return response+ return build_run_blocks_response(run_ok, result_data)async def _get_run_results(params: dict[str, Any], ctx: AgentContext) -> dict[str, Any]:diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.pyindex e130655d5..036c22fe0 100644--- a/skyvern/forge/sdk/schemas/workflow_copilot.py+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py@@ -142,6 +142,13 @@ class WorkflowCopilotToolResultUpdate(BaseModel):summary: str = Field(..., description="Brief human-readable summary of the result")iteration: int = Field(..., description="Agent loop iteration number")tool_call_id: str = Field(..., description="Unique ID for this tool invocation")+ detail: str | None = Field(+ None,+ description=(+ "Longer-cap sanitized failure text for tooltip display. None on success. "+ "Distinct from `summary`, which is capped tighter for the visible bullet."+ ),+ )class WorkflowCopilotCondensingUpdate(BaseModel):diff --git a/tests/unit/test_copilot_recoverable_failures.py b/tests/unit/test_copilot_recoverable_failures.pynew file mode 100644index 000000000..60f5c7a12--- /dev/null+++ b/tests/unit/test_copilot_recoverable_failures.py@@ -0,0 +1,153 @@+from __future__ import annotations++import json+from types import SimpleNamespace+from typing import Any+from unittest.mock import AsyncMock, MagicMock++import pytest+from agents.items import RunItem+from agents.stream_events import RunItemStreamEvent++from skyvern.forge.sdk.copilot.output_utils import (+ _sanitize_failure_text,+ build_run_blocks_response,+ sanitize_tool_result_for_llm,+ summarize_tool_result_detail,+)+from skyvern.forge.sdk.copilot.streaming_adapter import stream_to_sse+from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotStreamMessageType+++def test_summarize_tool_result_detail_returns_none_on_success() -> None:+ assert summarize_tool_result_detail({"ok": True, "data": {"block_count": 2}}) is None+++def test_summarize_tool_result_detail_caps_at_max_chars() -> None:+ long_error = "Workflow validation failed: " + ("missing field 'foo'; " * 200)+ detail = summarize_tool_result_detail({"ok": False, "error": long_error}, max_chars=400)+ assert detail is not None+ assert len(detail) <= 400+ assert detail.endswith("...")+++def test_summarize_tool_result_detail_preserves_short_full_message() -> None:+ detail = summarize_tool_result_detail(+ {"ok": False, "error": "Workflow validation failed: title field required"},+ )+ assert detail == "Workflow validation failed: title field required"+++def test_summarize_tool_result_detail_strips_header_blobs() -> None:+ text = "Failure with headers: {'host': 'x', 'authorization': 'Bearer abc'} please retry"+ detail = summarize_tool_result_detail({"ok": False, "error": text})+ assert detail is not None+ assert "authorization" not in detail+ assert "Bearer" not in detail+++def test_sanitize_failure_text_default_cap_unchanged() -> None:+ sanitized = _sanitize_failure_text("x" * 200)+ assert len(sanitized) == 120+ assert sanitized.endswith("...")+++def test_sanitize_failure_text_respects_max_chars() -> None:+ sanitized = _sanitize_failure_text("x" * 1000, max_chars=500)+ assert len(sanitized) == 500+ assert sanitized.endswith("...")+++def test_sanitize_tool_result_for_llm_passes_through_failure_dict() -> None:+ failure = {"ok": False, "error": "Workflow validation failed: title required"}+ sanitized = sanitize_tool_result_for_llm("update_workflow", failure)+ assert sanitized["ok"] is False+ assert sanitized["error"] == "Workflow validation failed: title required"+++def test_build_run_blocks_response_success_passes_through() -> None:+ response = build_run_blocks_response(True, {"workflow_run_id": "wr_test", "blocks": []})+ assert response == {"ok": True, "data": {"workflow_run_id": "wr_test", "blocks": []}}+++def test_build_run_blocks_response_promotes_run_level_failure_reason() -> None:+ response = build_run_blocks_response(+ False,+ {+ "workflow_run_id": "wr_test",+ "overall_status": "failed",+ "failure_reason": "Navigation timed out after 60s",+ "blocks": [],+ },+ )+ assert response["ok"] is False+ assert response["error"] == "Navigation timed out after 60s"+++def test_build_run_blocks_response_falls_back_when_no_failure_reason() -> None:+ response = build_run_blocks_response(False, {"workflow_run_id": "wr_test"})+ assert response["error"] == "Unknown error (no failure reason provided)"+++async def _capture_tool_result(tool_name: str, parsed_output: dict[str, Any]) -> Any:+ """Drive `stream_to_sse` over a single tool round-trip and return the+ emitted ``WorkflowCopilotToolResultUpdate``."""+ call_item = MagicMock(spec=RunItem)+ call_item.raw_item = {"call_id": "c1", "name": tool_name, "arguments": "{}"}+ tool_call = RunItemStreamEvent(name="tool_called", item=call_item)++ out_item = MagicMock(spec=RunItem)+ out_item.raw_item = {"call_id": "c1", "name": tool_name}+ out_item.output = [{"type": "text", "text": json.dumps(parsed_output)}]+ tool_output = RunItemStreamEvent(name="tool_output", item=out_item)++ async def _events() -> Any:+ yield tool_call+ yield tool_output++ result = MagicMock()+ result.stream_events = lambda: _events()+ result.cancel = MagicMock()++ sent: list[Any] = []++ async def _send(payload: Any) -> bool:+ sent.append(payload)+ return True++ stream = MagicMock()+ stream.is_disconnected = AsyncMock(return_value=False)+ stream.send = _send++ await stream_to_sse(result, stream, SimpleNamespace())++ tool_results = [p for p in sent if getattr(p, "type", None) == WorkflowCopilotStreamMessageType.TOOL_RESULT]+ assert len(tool_results) == 1+ return tool_results[0]+++async def test_stream_emits_detail_for_failure() -> None:+ long_error = "Workflow validation failed: " + (+ "blocks.0.task expects 'navigation_goal' but the emitted YAML omitted it. " * 4+ )+ payload = await _capture_tool_result(+ "update_workflow",+ {"ok": False, "error": long_error},+ )+ assert payload.success is False+ assert payload.detail is not None+ assert len(payload.detail) > 120+ # `summary` is the visible bullet, capped tighter than `detail` (the+ # tooltip-grade text) — strictly longer detail is the contract.+ assert len(payload.detail) > len(payload.summary)+++async def test_stream_emits_no_detail_for_success() -> None:+ payload = await _capture_tool_result(+ "update_workflow",+ {"ok": True, "data": {"block_count": 3}},+ )+ assert payload.success is True+ assert payload.detail is None