🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(in-app-agent): explain failed background runs without leaking internals (#16387)
Lysander Kiesel3e25f66eb1e2
diff --git a/web/src/features/in-app-agent/components/InAppAgentWindow.stories.tsx b/web/src/features/in-app-agent/components/InAppAgentWindow.stories.tsxindex b636627d1d..d25b3eb1eb 100644--- a/web/src/features/in-app-agent/components/InAppAgentWindow.stories.tsx+++ b/web/src/features/in-app-agent/components/InAppAgentWindow.stories.tsx@@ -1602,7 +1602,10 @@ export const Failed = meta.story({const canvas = within(canvasElement);await expect(canvas.getByRole("status")).toHaveTextContent(- "The run exceeded the maximum duration.",+ "The run hit the time limit.",+ );+ await expect(canvas.getByRole("status")).toHaveTextContent(+ "Send another message",);await expect(canvas.getByRole("button", { name: "Failed after 51s" }),@@ -1672,7 +1675,10 @@ export const FailedBeforeFirstToken = meta.story({const canvas = within(canvasElement);await expect(canvas.getByRole("status")).toHaveTextContent(- "No worker picked this up",+ "The assistant failed.",+ );+ await expect(canvas.getByRole("status")).toHaveTextContent(+ "Send another message",);await expect(canvas.getByRole("button", { name: "Worked for 12s" }),@@ -1683,6 +1689,126 @@ export const FailedBeforeFirstToken = meta.story({},});+const failedWorkerLostRun = {+ id: "run-1",+ status: InAppAgentRunStatus.FAILED,+ errorCode: InAppAgentRunErrorCode.WORKER_LOST,+ cancelRequested: false,+};++export const FailedWorkerLost = meta.story({+ name: "(Test) Assistant failed",+ args: {+ selectedConversationId: "conversation-1",+ executionUi: {+ notice: getBackgroundRunNotice(failedWorkerLostRun),+ activityOutcome: getSettledActivityOutcome(failedWorkerLostRun),+ stop: null,+ },+ messages: [+ {+ id: "user-1",+ role: "user",+ content: {+ type: "text",+ text: "Investigate latency",+ },+ },+ {+ id: "assistant-answer",+ runId: "run-1",+ timestamp: new Date("2026-08-06T15:27:17.000Z").getTime(),+ role: "assistant",+ content: {+ type: "text",+ text: "Still inspecting the remaining traces.",+ },+ },+ ],+ },+ play: async ({ canvasElement }) => {+ const canvas = within(canvasElement);++ await expect(canvas.getByRole("status")).toHaveTextContent(+ "The assistant failed.",+ );+ await expect(canvas.getByRole("status")).toHaveTextContent(+ "Send another message",+ );+ await expect(+ canvas.getByRole("textbox", { name: "Message the assistant" }),+ ).toBeEnabled();+ },+});++const expiredApprovalRun = {+ id: "run-1",+ status: InAppAgentRunStatus.FAILED,+ errorCode: InAppAgentRunErrorCode.APPROVAL_EXPIRED,+ cancelRequested: false,+};++export const ApprovalExpired = meta.story({+ name: "(Test) Approval expired",+ args: {+ selectedConversationId: "conversation-1",+ isAwaitingApproval: false,+ isAssistantTurnInProgress: false,+ executionUi: {+ notice: getBackgroundRunNotice(expiredApprovalRun),+ activityOutcome: getSettledActivityOutcome(expiredApprovalRun),+ stop: null,+ },+ messages: [+ {+ id: "user-1",+ role: "user",+ content: {+ type: "text",+ text: "Create a dataset for regression examples.",+ },+ },+ {+ id: "assistant-tool-1",+ role: "assistant",+ content: {+ type: "toolGroup",+ tools: [+ {+ type: "tool",+ name: "langfuse_upsertDataset",+ status: "running",+ args: JSON.stringify({+ name: "regression-examples",+ description: "Examples used for release regression tests",+ }),+ },+ ],+ },+ },+ ],+ },+ play: async ({ canvasElement }) => {+ const canvas = within(canvasElement);++ await expect(canvas.getByRole("status")).toHaveTextContent(+ "The approval request expired. The action was not run. Send another message if you still want it.",+ );+ await expect(+ canvas.queryByRole("button", { name: "Approve" }),+ ).not.toBeInTheDocument();+ await expect(+ canvas.queryByRole("button", { name: "Decline" }),+ ).not.toBeInTheDocument();+ await expect(+ canvas.queryByRole("button", { name: "Waiting for your approval…" }),+ ).not.toBeInTheDocument();+ await expect(+ canvas.getByRole("textbox", { name: "Message the assistant" }),+ ).toBeEnabled();+ },+});+/*** Every activity state in the row it belongs to, sharing one fixed-width slot* so the column stays straight despite the dots being narrower than the icons.diff --git a/web/src/features/in-app-agent/lib/backgroundExecutionSession.clienttest.ts b/web/src/features/in-app-agent/lib/backgroundExecutionSession.clienttest.tsindex 1cfeaa2419..b7b59d0495 100644--- a/web/src/features/in-app-agent/lib/backgroundExecutionSession.clienttest.ts+++ b/web/src/features/in-app-agent/lib/backgroundExecutionSession.clienttest.ts@@ -17,6 +17,7 @@ import {import { BackgroundExecutionConnectionError } from "./backgroundExecutionErrors";import {BackgroundExecutionSessionController,+ getBackgroundRunNotice,type BackgroundExecutionView,} from "./backgroundExecutionSession";@@ -1364,6 +1365,48 @@ describe("InAppAgentBackgroundClient reconnect", () => {});});+describe("getBackgroundRunNotice", () => {+ const failedRun = (errorCode: string | null) => ({+ id: "run-1",+ status: InAppAgentRunStatus.FAILED,+ errorCode,+ cancelRequested: false,+ });++ const assistantFailedContinue =+ "The assistant failed. Send another message to continue.";+ const assistantFailedTryAgain =+ "The assistant failed. Send another message to try again.";++ it.each([+ [InAppAgentRunErrorCode.WORKER_LOST, assistantFailedContinue],+ [InAppAgentRunErrorCode.STALE, assistantFailedContinue],+ [InAppAgentRunErrorCode.QUEUE_TIMEOUT, assistantFailedContinue],+ [InAppAgentRunErrorCode.WORKER_SHUTDOWN, assistantFailedContinue],+ [InAppAgentRunErrorCode.OUTCOME_UNKNOWN, assistantFailedContinue],+ [InAppAgentRunErrorCode.INIT_FAILED, assistantFailedTryAgain],+ [InAppAgentRunErrorCode.ENQUEUE_FAILED, assistantFailedTryAgain],+ [+ InAppAgentRunErrorCode.APPROVAL_EXPIRED,+ "The approval request expired. The action was not run. Send another message if you still want it.",+ ],+ [+ InAppAgentRunErrorCode.RUN_TIMEOUT,+ "The run hit the time limit. Send another message to continue.",+ ],+ [+ InAppAgentRunErrorCode.AGENT_ERROR,+ "The assistant hit an error before finishing. Send another message to continue.",+ ],+ ["mystery_code", "The run failed. Try again."],+ ] as const)("maps FAILED %s to the user-facing notice", (errorCode, text) => {+ expect(getBackgroundRunNotice(failedRun(errorCode))).toEqual({+ text,+ tone: "info",+ });+ });+});+function sseFrame(frame: Record<string, unknown>): string {return `event: ${String(frame.type)}\ndata: ${JSON.stringify(frame)}\n\n`;}diff --git a/web/src/features/in-app-agent/lib/backgroundExecutionSession.ts b/web/src/features/in-app-agent/lib/backgroundExecutionSession.tsindex 6617e949aa..9a3f7a7fad 100644--- a/web/src/features/in-app-agent/lib/backgroundExecutionSession.ts+++ b/web/src/features/in-app-agent/lib/backgroundExecutionSession.ts@@ -567,6 +567,30 @@ export function isCancellableBackgroundRun();}+const ASSISTANT_FAILED_CONTINUE =+ "The assistant failed. Send another message to continue.";+const ASSISTANT_FAILED_TRY_AGAIN =+ "The assistant failed. Send another message to try again.";++const BACKGROUND_RUN_FAILURE_MESSAGES: Readonly<Record<string, string>> = {+ [InAppAgentRunErrorCode.WORKER_LOST]: ASSISTANT_FAILED_CONTINUE,+ [InAppAgentRunErrorCode.STALE]: ASSISTANT_FAILED_CONTINUE,+ [InAppAgentRunErrorCode.QUEUE_TIMEOUT]: ASSISTANT_FAILED_CONTINUE,+ [InAppAgentRunErrorCode.WORKER_SHUTDOWN]: ASSISTANT_FAILED_CONTINUE,+ [InAppAgentRunErrorCode.OUTCOME_UNKNOWN]: ASSISTANT_FAILED_CONTINUE,+ [InAppAgentRunErrorCode.INIT_FAILED]: ASSISTANT_FAILED_TRY_AGAIN,+ [InAppAgentRunErrorCode.ENQUEUE_FAILED]: ASSISTANT_FAILED_TRY_AGAIN,+ [InAppAgentRunErrorCode.APPROVAL_EXPIRED]:+ "The approval request expired. The action was not run. Send another message if you still want it.",+ [InAppAgentRunErrorCode.RUN_TIMEOUT]:+ "The run hit the time limit. Send another message to continue.",+ [InAppAgentRunErrorCode.AGENT_ERROR]:+ "The assistant hit an error before finishing. Send another message to continue.",+ [InAppAgentRunErrorCode.APPROVAL_SUPERSEDED]: "Replaced by a newer message.",+ [InAppAgentRunErrorCode.APPROVAL_CANCELLED]: "Approval cancelled.",+ [InAppAgentRunErrorCode.CANCELLED]: "You stopped this run.",+};+export function getBackgroundRunFailureMessage(errorCode: string | null,): string {@@ -638,24 +662,6 @@ export function getSettledActivityOutcome(return "worked";}-const BACKGROUND_RUN_FAILURE_MESSAGES: Readonly<Record<string, string>> = {- [InAppAgentRunErrorCode.ENQUEUE_FAILED]: "Couldn't start the run. Try again.",- [InAppAgentRunErrorCode.QUEUE_TIMEOUT]:- "No worker picked this up. Try again.",- [InAppAgentRunErrorCode.WORKER_LOST]: "The run was interrupted. Try again.",- [InAppAgentRunErrorCode.STALE]: "The run was interrupted. Try again.",- [InAppAgentRunErrorCode.RUN_TIMEOUT]:- "The run exceeded the maximum duration.",- [InAppAgentRunErrorCode.WORKER_SHUTDOWN]:- "The run was interrupted by a deploy. Try again.",- [InAppAgentRunErrorCode.OUTCOME_UNKNOWN]:- "The approved action may have completed. Verify before retrying.",- [InAppAgentRunErrorCode.APPROVAL_EXPIRED]: "The approval request expired.",- [InAppAgentRunErrorCode.APPROVAL_SUPERSEDED]: "Replaced by a newer message.",- [InAppAgentRunErrorCode.APPROVAL_CANCELLED]: "Approval cancelled.",- [InAppAgentRunErrorCode.CANCELLED]: "You stopped this run.",-};-function isExecutingRun(run: BackgroundExecutionRunView | null,): run is BackgroundExecutionRunView {