Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(in-app-agent): explain failed background runs without leaking internals (#16387)

Lysander Kiesel3e25f66eb1e2
diff --git a/web/src/features/in-app-agent/components/InAppAgentWindow.stories.tsx b/web/src/features/in-app-agent/components/InAppAgentWindow.stories.tsx
index b636627d1d..d25b3eb1eb 100644
--- a/web/src/features/in-app-agent/components/InAppAgentWindow.stories.tsx
+++ b/web/src/features/in-app-agent/components/InAppAgentWindow.stories.tsx
@@ -1602,7 +1602,10 @@ export const Failed = meta.story({
const canvas = within(canvasElement);
await expect(canvas.getByRole("status")).toHaveTextContent(
- "The run exceeded the maximum duration.",
+ "The run hit the time limit.",
+ );
+ await expect(canvas.getByRole("status")).toHaveTextContent(
+ "Send another message",
);
await expect(
canvas.getByRole("button", { name: "Failed after 51s" }),
@@ -1672,7 +1675,10 @@ export const FailedBeforeFirstToken = meta.story({
const canvas = within(canvasElement);
await expect(canvas.getByRole("status")).toHaveTextContent(
- "No worker picked this up",
+ "The assistant failed.",
+ );
+ await expect(canvas.getByRole("status")).toHaveTextContent(
+ "Send another message",
);
await expect(
canvas.getByRole("button", { name: "Worked for 12s" }),
@@ -1683,6 +1689,126 @@ export const FailedBeforeFirstToken = meta.story({
},
});
+const failedWorkerLostRun = {
+ id: "run-1",
+ status: InAppAgentRunStatus.FAILED,
+ errorCode: InAppAgentRunErrorCode.WORKER_LOST,
+ cancelRequested: false,
+};
+
+export const FailedWorkerLost = meta.story({
+ name: "(Test) Assistant failed",
+ args: {
+ selectedConversationId: "conversation-1",
+ executionUi: {
+ notice: getBackgroundRunNotice(failedWorkerLostRun),
+ activityOutcome: getSettledActivityOutcome(failedWorkerLostRun),
+ stop: null,
+ },
+ messages: [
+ {
+ id: "user-1",
+ role: "user",
+ content: {
+ type: "text",
+ text: "Investigate latency",
+ },
+ },
+ {
+ id: "assistant-answer",
+ runId: "run-1",
+ timestamp: new Date("2026-08-06T15:27:17.000Z").getTime(),
+ role: "assistant",
+ content: {
+ type: "text",
+ text: "Still inspecting the remaining traces.",
+ },
+ },
+ ],
+ },
+ play: async ({ canvasElement }) => {
+ const canvas = within(canvasElement);
+
+ await expect(canvas.getByRole("status")).toHaveTextContent(
+ "The assistant failed.",
+ );
+ await expect(canvas.getByRole("status")).toHaveTextContent(
+ "Send another message",
+ );
+ await expect(
+ canvas.getByRole("textbox", { name: "Message the assistant" }),
+ ).toBeEnabled();
+ },
+});
+
+const expiredApprovalRun = {
+ id: "run-1",
+ status: InAppAgentRunStatus.FAILED,
+ errorCode: InAppAgentRunErrorCode.APPROVAL_EXPIRED,
+ cancelRequested: false,
+};
+
+export const ApprovalExpired = meta.story({
+ name: "(Test) Approval expired",
+ args: {
+ selectedConversationId: "conversation-1",
+ isAwaitingApproval: false,
+ isAssistantTurnInProgress: false,
+ executionUi: {
+ notice: getBackgroundRunNotice(expiredApprovalRun),
+ activityOutcome: getSettledActivityOutcome(expiredApprovalRun),
+ stop: null,
+ },
+ messages: [
+ {
+ id: "user-1",
+ role: "user",
+ content: {
+ type: "text",
+ text: "Create a dataset for regression examples.",
+ },
+ },
+ {
+ id: "assistant-tool-1",
+ role: "assistant",
+ content: {
+ type: "toolGroup",
+ tools: [
+ {
+ type: "tool",
+ name: "langfuse_upsertDataset",
+ status: "running",
+ args: JSON.stringify({
+ name: "regression-examples",
+ description: "Examples used for release regression tests",
+ }),
+ },
+ ],
+ },
+ },
+ ],
+ },
+ play: async ({ canvasElement }) => {
+ const canvas = within(canvasElement);
+
+ await expect(canvas.getByRole("status")).toHaveTextContent(
+ "The approval request expired. The action was not run. Send another message if you still want it.",
+ );
+ await expect(
+ canvas.queryByRole("button", { name: "Approve" }),
+ ).not.toBeInTheDocument();
+ await expect(
+ canvas.queryByRole("button", { name: "Decline" }),
+ ).not.toBeInTheDocument();
+ await expect(
+ canvas.queryByRole("button", { name: "Waiting for your approval…" }),
+ ).not.toBeInTheDocument();
+ await expect(
+ canvas.getByRole("textbox", { name: "Message the assistant" }),
+ ).toBeEnabled();
+ },
+});
+
/**
* Every activity state in the row it belongs to, sharing one fixed-width slot
* so the column stays straight despite the dots being narrower than the icons.
diff --git a/web/src/features/in-app-agent/lib/backgroundExecutionSession.clienttest.ts b/web/src/features/in-app-agent/lib/backgroundExecutionSession.clienttest.ts
index 1cfeaa2419..b7b59d0495 100644
--- a/web/src/features/in-app-agent/lib/backgroundExecutionSession.clienttest.ts
+++ b/web/src/features/in-app-agent/lib/backgroundExecutionSession.clienttest.ts
@@ -17,6 +17,7 @@ import {
import { BackgroundExecutionConnectionError } from "./backgroundExecutionErrors";
import {
BackgroundExecutionSessionController,
+ getBackgroundRunNotice,
type BackgroundExecutionView,
} from "./backgroundExecutionSession";
@@ -1364,6 +1365,48 @@ describe("InAppAgentBackgroundClient reconnect", () => {
});
});
+describe("getBackgroundRunNotice", () => {
+ const failedRun = (errorCode: string | null) => ({
+ id: "run-1",
+ status: InAppAgentRunStatus.FAILED,
+ errorCode,
+ cancelRequested: false,
+ });
+
+ const assistantFailedContinue =
+ "The assistant failed. Send another message to continue.";
+ const assistantFailedTryAgain =
+ "The assistant failed. Send another message to try again.";
+
+ it.each([
+ [InAppAgentRunErrorCode.WORKER_LOST, assistantFailedContinue],
+ [InAppAgentRunErrorCode.STALE, assistantFailedContinue],
+ [InAppAgentRunErrorCode.QUEUE_TIMEOUT, assistantFailedContinue],
+ [InAppAgentRunErrorCode.WORKER_SHUTDOWN, assistantFailedContinue],
+ [InAppAgentRunErrorCode.OUTCOME_UNKNOWN, assistantFailedContinue],
+ [InAppAgentRunErrorCode.INIT_FAILED, assistantFailedTryAgain],
+ [InAppAgentRunErrorCode.ENQUEUE_FAILED, assistantFailedTryAgain],
+ [
+ InAppAgentRunErrorCode.APPROVAL_EXPIRED,
+ "The approval request expired. The action was not run. Send another message if you still want it.",
+ ],
+ [
+ InAppAgentRunErrorCode.RUN_TIMEOUT,
+ "The run hit the time limit. Send another message to continue.",
+ ],
+ [
+ InAppAgentRunErrorCode.AGENT_ERROR,
+ "The assistant hit an error before finishing. Send another message to continue.",
+ ],
+ ["mystery_code", "The run failed. Try again."],
+ ] as const)("maps FAILED %s to the user-facing notice", (errorCode, text) => {
+ expect(getBackgroundRunNotice(failedRun(errorCode))).toEqual({
+ text,
+ tone: "info",
+ });
+ });
+});
+
function sseFrame(frame: Record<string, unknown>): string {
return `event: ${String(frame.type)}\ndata: ${JSON.stringify(frame)}\n\n`;
}
diff --git a/web/src/features/in-app-agent/lib/backgroundExecutionSession.ts b/web/src/features/in-app-agent/lib/backgroundExecutionSession.ts
index 6617e949aa..9a3f7a7fad 100644
--- a/web/src/features/in-app-agent/lib/backgroundExecutionSession.ts
+++ b/web/src/features/in-app-agent/lib/backgroundExecutionSession.ts
@@ -567,6 +567,30 @@ export function isCancellableBackgroundRun(
);
}
+const ASSISTANT_FAILED_CONTINUE =
+ "The assistant failed. Send another message to continue.";
+const ASSISTANT_FAILED_TRY_AGAIN =
+ "The assistant failed. Send another message to try again.";
+
+const BACKGROUND_RUN_FAILURE_MESSAGES: Readonly<Record<string, string>> = {
+ [InAppAgentRunErrorCode.WORKER_LOST]: ASSISTANT_FAILED_CONTINUE,
+ [InAppAgentRunErrorCode.STALE]: ASSISTANT_FAILED_CONTINUE,
+ [InAppAgentRunErrorCode.QUEUE_TIMEOUT]: ASSISTANT_FAILED_CONTINUE,
+ [InAppAgentRunErrorCode.WORKER_SHUTDOWN]: ASSISTANT_FAILED_CONTINUE,
+ [InAppAgentRunErrorCode.OUTCOME_UNKNOWN]: ASSISTANT_FAILED_CONTINUE,
+ [InAppAgentRunErrorCode.INIT_FAILED]: ASSISTANT_FAILED_TRY_AGAIN,
+ [InAppAgentRunErrorCode.ENQUEUE_FAILED]: ASSISTANT_FAILED_TRY_AGAIN,
+ [InAppAgentRunErrorCode.APPROVAL_EXPIRED]:
+ "The approval request expired. The action was not run. Send another message if you still want it.",
+ [InAppAgentRunErrorCode.RUN_TIMEOUT]:
+ "The run hit the time limit. Send another message to continue.",
+ [InAppAgentRunErrorCode.AGENT_ERROR]:
+ "The assistant hit an error before finishing. Send another message to continue.",
+ [InAppAgentRunErrorCode.APPROVAL_SUPERSEDED]: "Replaced by a newer message.",
+ [InAppAgentRunErrorCode.APPROVAL_CANCELLED]: "Approval cancelled.",
+ [InAppAgentRunErrorCode.CANCELLED]: "You stopped this run.",
+};
+
export function getBackgroundRunFailureMessage(
errorCode: string | null,
): string {
@@ -638,24 +662,6 @@ export function getSettledActivityOutcome(
return "worked";
}
-const BACKGROUND_RUN_FAILURE_MESSAGES: Readonly<Record<string, string>> = {
- [InAppAgentRunErrorCode.ENQUEUE_FAILED]: "Couldn't start the run. Try again.",
- [InAppAgentRunErrorCode.QUEUE_TIMEOUT]:
- "No worker picked this up. Try again.",
- [InAppAgentRunErrorCode.WORKER_LOST]: "The run was interrupted. Try again.",
- [InAppAgentRunErrorCode.STALE]: "The run was interrupted. Try again.",
- [InAppAgentRunErrorCode.RUN_TIMEOUT]:
- "The run exceeded the maximum duration.",
- [InAppAgentRunErrorCode.WORKER_SHUTDOWN]:
- "The run was interrupted by a deploy. Try again.",
- [InAppAgentRunErrorCode.OUTCOME_UNKNOWN]:
- "The approved action may have completed. Verify before retrying.",
- [InAppAgentRunErrorCode.APPROVAL_EXPIRED]: "The approval request expired.",
- [InAppAgentRunErrorCode.APPROVAL_SUPERSEDED]: "Replaced by a newer message.",
- [InAppAgentRunErrorCode.APPROVAL_CANCELLED]: "Approval cancelled.",
- [InAppAgentRunErrorCode.CANCELLED]: "You stopped this run.",
-};
-
function isExecutingRun(
run: BackgroundExecutionRunView | null,
): run is BackgroundExecutionRunView {