π OpenHands: AI-Driven Development
feat(canvas): present structured error outcomes (#16231)
Graham Neubig0d15c5e79e91
diff --git a/__tests__/components/chat/error-message-banner.test.tsx b/__tests__/components/chat/error-message-banner.test.tsxindex 1535d793dd..ede6142475 100644--- a/__tests__/components/chat/error-message-banner.test.tsx+++ b/__tests__/components/chat/error-message-banner.test.tsx@@ -54,6 +54,57 @@ describe("ErrorMessageBanner", () => {expect(icon).toHaveStyle({ color: "var(--oh-status-error)" });});+ it("uses a warning icon for non-internal outcomes", () => {+ render(+ <ErrorMessageBanner+ message="Incorrect API key"+ classification={{+ kind: "auth",+ retryable: false,+ user_action: "settings",+ }}+ />,+ );++ expect(screen.getByTestId("warning-message-banner-icon")).toHaveClass(+ "text-[var(--oh-warning)]",+ );+ });++ it("uses the error icon for internal outcomes", () => {+ render(+ <ErrorMessageBanner+ message="Something went wrong"+ classification={{+ kind: "internal",+ retryable: false,+ user_action: "none",+ }}+ />,+ );++ expect(screen.getByTestId("error-message-banner-icon")).toHaveStyle({+ color: "var(--oh-status-error)",+ });+ });++ it("uses the error icon for unknown (diagnostic) outcomes", () => {+ render(+ <ErrorMessageBanner+ message="Something went wrong"+ classification={{+ kind: "unknown",+ retryable: false,+ user_action: "none",+ }}+ />,+ );++ expect(screen.getByTestId("error-message-banner-icon")).toHaveStyle({+ color: "var(--oh-status-error)",+ });+ });+it("uses greyscale theme tokens instead of red error styling", () => {render(<ErrorMessageBanner message="Something went wrong" />);diff --git a/__tests__/contexts/conversation-websocket-context.test.tsx b/__tests__/contexts/conversation-websocket-context.test.tsxindex 1147b8dd10..05dd3c113f 100644--- a/__tests__/contexts/conversation-websocket-context.test.tsx+++ b/__tests__/contexts/conversation-websocket-context.test.tsx@@ -27,12 +27,17 @@ type CapturedWebSocketOptions = {const wsCapture = vi.hoisted(() => ({mainOnMessage: null as null | ((event: { data: string }) => void),mainOptions: null as CapturedWebSocketOptions | null,+ planningOnMessage: null as null | ((event: { data: string }) => void),calls: [] as Array<{url: string;options?: CapturedWebSocketOptions;}>,}));+const errorHandlerMocks = vi.hoisted(() => ({+ trackError: vi.fn(),+}));+// Keep the units under test real (the provider, `useConversationHistory`, the// event store). Only the network is stubbed: the WebSocket transport and the// REST service the history query depends on.@@ -50,12 +55,23 @@ vi.mock("#/hooks/use-websocket", () => ({wsCapture.mainOnMessage = options.onMessage;wsCapture.mainOptions = options;}+ if (+ url &&+ options?.onMessage &&+ options.queryParams &&+ "resend_all" in options.queryParams+ ) {+ wsCapture.planningOnMessage = options.onMessage;+ }return { socket: null, reconnect: vi.fn() };}),}));vi.mock("#/hooks/query/use-user-conversation", () => ({useUserConversation: vi.fn(),}));+vi.mock("#/utils/error-handler", () => ({+ trackError: errorHandlerMocks.trackError,+}));const AGENT_REPLY_ID = "evt-agent-reply";@@ -91,6 +107,7 @@ describe("ConversationWebSocketProvider β conversation-scoped event store", ()beforeEach(() => {wsCapture.mainOnMessage = null;wsCapture.mainOptions = null;+ wsCapture.planningOnMessage = null;wsCapture.calls.length = 0;window.localStorage.clear();queryClient = new QueryClient({@@ -356,13 +373,42 @@ describe("ConversationWebSocketProvider β conversation-scoped event store", ()},});- const makeConversationError = (id: string, detail: string) => ({+ const makeConversationError = (+ id: string,+ detail: string,+ classification?: {+ kind: "auth";+ retryable: boolean;+ user_action: "settings";+ },+ ) => ({id,timestamp: new Date().toISOString(),source: "environment",kind: "ConversationErrorEvent",detail,code: "SomeError",+ ...(classification ? { classification } : {}),+ });++ const makeAgentError = (+ id: string,+ classification?: {+ kind: "auth";+ retryable: boolean;+ user_action: "settings";+ },+ ) => ({+ id,+ timestamp: new Date().toISOString(),+ source: "agent",+ message_id: `msg-${id}`,+ message_seq: 1,+ error: "Agent failed",+ error_type: "AgentError",+ tool_name: "generic",+ tool_call_id: `call-${id}`,+ ...(classification ? { classification } : {}),});const renderCaptured = async () => {@@ -421,6 +467,115 @@ describe("ConversationWebSocketProvider β conversation-scoped event store", ()deliver(errorEvent);expect(useErrorMessageStore.getState().errorMessage).toBeNull();});++ it("forwards error classifications to the banner store and telemetry", async () => {+ await renderCaptured();+ const classification = {+ kind: "auth" as const,+ retryable: false,+ user_action: "settings" as const,+ };++ deliver(+ makeConversationError(+ "conv-error-2",+ "Authentication failed",+ classification,+ ),+ );++ expect(useErrorMessageStore.getState().errorClassification).toEqual(+ classification,+ );+ expect(errorHandlerMocks.trackError).toHaveBeenCalledWith({+ source: "conversation",+ metadata: {+ eventId: "conv-error-2",+ errorCode: "SomeError",+ },+ classification,+ });+ });++ it("forwards AgentErrorEvent classifications to telemetry (main agent)", async () => {+ await renderCaptured();+ const classification = {+ kind: "auth" as const,+ retryable: false,+ user_action: "settings" as const,+ };++ deliver(makeAgentError("agent-err-1", classification));++ expect(errorHandlerMocks.trackError).toHaveBeenCalledWith({+ source: "agent",+ metadata: {+ eventId: "agent-err-1",+ toolName: "generic",+ toolCallId: "call-agent-err-1",+ },+ classification,+ });+ });++ it("forwards AgentErrorEvent classifications to telemetry (planning agent)", async () => {+ const planningConversation: AppConversation = {+ id: "planning-err",+ created_by_user_id: null,+ selected_repository: null,+ selected_branch: null,+ git_provider: null,+ title: "Planner",+ trigger: null,+ pr_number: [],+ llm_model: null,+ metrics: null,+ created_at: "2026-07-28T00:00:00Z",+ updated_at: "2026-07-28T00:00:00Z",+ execution_status: null,+ conversation_url:+ "http://planner.example/api/conversations/planning-err",+ session_api_key: null,+ sandbox_id: null,+ sub_conversation_ids: [],+ };+ const classification = {+ kind: "auth" as const,+ retryable: true,+ user_action: "settings" as const,+ };++ render(+ <QueryClientProvider client={queryClient}>+ <ConversationWebSocketProvider+ conversationId="conv-err"+ conversationUrl="http://main.example/api/conversations/conv-err"+ subConversationIds={[planningConversation.id]}+ subConversations={[planningConversation]}+ >+ <div />+ </ConversationWebSocketProvider>+ </QueryClientProvider>,+ );+ // Wait for the planning sub-conversation WebSocket to be established.+ await waitFor(() => expect(wsCapture.planningOnMessage).not.toBeNull());++ act(() => {+ wsCapture.planningOnMessage!({+ data: JSON.stringify(makeAgentError("agent-err-2", classification)),+ });+ });++ expect(errorHandlerMocks.trackError).toHaveBeenCalledWith({+ source: "planning_agent",+ metadata: {+ eventId: "agent-err-2",+ toolName: "generic",+ toolCallId: "call-agent-err-2",+ },+ classification,+ });+ });});it("clears the previous conversation's events when switching conversations", async () => {diff --git a/__tests__/stores/error-message-store.test.ts b/__tests__/stores/error-message-store.test.tsindex f768424275..6b19812747 100644--- a/__tests__/stores/error-message-store.test.ts+++ b/__tests__/stores/error-message-store.test.ts@@ -9,6 +9,7 @@ describe("error message store", () => {errorMessage: null,errorType: null,errorCode: null,+ errorClassification: null,});});@@ -58,6 +59,25 @@ describe("error message store", () => {expect(getState().errorCode).toBe("ACPAuthRequired");});+ it("stores and clears structured error classifications", () => {+ const classification = {+ kind: "auth" as const,+ retryable: false,+ user_action: "settings" as const,+ };++ getState().setErrorMessage(+ "auth failed",+ "conversation",+ null,+ classification,+ );+ expect(getState().errorClassification).toEqual(classification);++ getState().removeErrorMessage();+ expect(getState().errorClassification).toBeNull();+ });+it("removeErrorMessage clears the error code too", () => {getState().setErrorMessage("auth failed", "conversation", "ACPAuthRequired");getState().removeErrorMessage();diff --git a/__tests__/utils/error-handler.test.ts b/__tests__/utils/error-handler.test.tsindex 3aba0ef374..bbc1b1d167 100644--- a/__tests__/utils/error-handler.test.ts+++ b/__tests__/utils/error-handler.test.ts@@ -1,12 +1,15 @@import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";import { trackError } from "#/utils/error-handler";-import { trackException } from "#/services/telemetry";+import { trackEvent } from "#/services/telemetry";vi.mock("#/services/telemetry", () => ({- trackException: vi.fn(),+ trackEvent: vi.fn(),}));describe("Error Handler", () => {+ beforeEach(() => {+ window.history.replaceState({}, "", "/test-error");+ });beforeEach(() => {vi.clearAllMocks();});@@ -18,40 +21,122 @@ describe("Error Handler", () => {describe("trackError", () => {it("should send error to PostHog with basic info", () => {const error = {- message: "Test error",source: "test",};trackError(error);- expect(trackException).toHaveBeenCalledWith(- new Error("Test error"),- {- error_source: "test",- },- );+ expect(trackEvent).toHaveBeenCalledWith("error_outcome", {+ current_url: window.location.href,+ error_source: "test",+ error_kind: "unknown",+ error_telemetry: "diagnostic",+ });});- it("should include additional metadata in PostHog event", () => {+ it("merges metadata while reserving error outcome fields", () => {const error = {- message: "Test error",source: "test",metadata: {extra: "info",details: { foo: "bar" },+ error_kind: "spoofed",+ error_telemetry: "outcome",+ error_id: "spoofed",},};trackError(error);- expect(trackException).toHaveBeenCalledWith(- new Error("Test error"),- {- error_source: "test",- extra: "info",- details: { foo: "bar" },+ expect(trackEvent).toHaveBeenCalledWith("error_outcome", {+ current_url: window.location.href,+ error_source: "test",+ error_kind: "unknown",+ error_telemetry: "diagnostic",+ extra: "info",+ details: { foo: "bar" },+ });+ });++ it("keeps a classified error ID without recording the message", () => {+ trackError({+ source: "agent",+ classification: {+ kind: "internal",+ retryable: false,+ user_action: "none",+ error_id: "error-123",},+ });++ expect(trackEvent).toHaveBeenCalledWith("error_outcome", {+ current_url: window.location.href,+ error_source: "agent",+ error_kind: "internal",+ error_id: "error-123",+ error_telemetry: "diagnostic",+ });+ });++ it("promotes a string metadata eventId as the error_id fallback", () => {+ trackError({+ source: "chat",+ metadata: { msgId: "m-1", eventId: "evt-42", extra: "info" },+ });++ expect(trackEvent).toHaveBeenCalledWith("error_outcome", {+ current_url: window.location.href,+ error_source: "chat",+ error_kind: "unknown",+ error_id: "evt-42",+ error_telemetry: "diagnostic",+ msgId: "m-1",+ extra: "info",+ });+ });++ it("lets a classification error_id win over a metadata eventId", () => {+ trackError({+ source: "agent",+ metadata: { msgId: "m-1", eventId: "evt-42" },+ classification: {+ kind: "auth",+ retryable: false,+ user_action: "settings",+ error_id: "cls-7",+ },+ });++ expect(trackEvent).toHaveBeenCalledWith(+ "error_outcome",+ expect.objectContaining({+ current_url: window.location.href,+ error_id: "cls-7",+ error_telemetry: "outcome",+ }),);+ expect(trackEvent).not.toHaveBeenCalledWith(+ "error_outcome",+ expect.objectContaining({ eventId: "evt-42" }),+ );+ });++ it("records non-internal classifications as outcomes", () => {+ trackError({+ source: "agent",+ classification: {+ kind: "auth",+ retryable: false,+ user_action: "settings",+ },+ });++ expect(trackEvent).toHaveBeenCalledWith("error_outcome", {+ current_url: window.location.href,+ error_source: "agent",+ error_kind: "auth",+ error_telemetry: "outcome",+ });});});});diff --git a/src/components/features/chat/chat-interface.tsx b/src/components/features/chat/chat-interface.tsxindex 72c80ba4ec..5521222962 100644--- a/src/components/features/chat/chat-interface.tsx+++ b/src/components/features/chat/chat-interface.tsx@@ -64,8 +64,13 @@ function getEntryPoint(export function ChatInterface() {const { trackInitialQuerySubmitted, trackUserMessageSent } = useTracking();const { setMessageToSend } = useConversationStore();- const { errorMessage, errorCode, removeErrorMessage, setErrorMessage } =- useErrorMessageStore();+ const {+ errorMessage,+ errorCode,+ errorClassification,+ removeErrorMessage,+ setErrorMessage,+ } = useErrorMessageStore();const navigate = useNavigate();const { isTask, taskStatus, taskDetail } = useTaskPolling();// Hide empty-state chrome for the entire `/conversations/task-{uuid}` route,@@ -570,6 +575,7 @@ export function ChatInterface() {<ErrorMessageBannermessage={errorMessage}code={errorCode}+ classification={errorClassification}onDismiss={removeErrorMessage}onRetry={errorMessage === SERVER_CONNECTION_ERROR_MESSAGEdiff --git a/src/components/features/chat/error-message-banner.tsx b/src/components/features/chat/error-message-banner.tsxindex 888f0fa116..cff1972000 100644--- a/src/components/features/chat/error-message-banner.tsx+++ b/src/components/features/chat/error-message-banner.tsx@@ -1,11 +1,12 @@import React from "react";import { Trans, useTranslation } from "react-i18next";-import { Check, CircleX, Copy, X } from "lucide-react";+import { Check, CircleAlert, CircleX, Copy, X } from "lucide-react";import { OH_STATUS_ERROR_COLOR } from "#/constants/status-colors";import { I18nKey } from "#/i18n/declaration";import { displayErrorToast } from "#/utils/custom-toast-handlers";import { getAcpErrorHeaderKey } from "#/utils/acp-error-codes";import { cn } from "#/utils/utils";+import type { ErrorClassification } from "@openhands/typescript-client";interface ErrorMessageBannerProps {message: string;@@ -15,6 +16,7 @@ interface ErrorMessageBannerProps {onRetry?: () => void;/** Recovery action (e.g. re-authenticate) shown for credential failures. */onReauth?: () => void;+ classification?: ErrorClassification | null;}const DEFAULT_MAX_COLLAPSED_CHARS = 220;@@ -25,6 +27,7 @@ export function ErrorMessageBanner({onDismiss,onRetry,onReauth,+ classification,}: ErrorMessageBannerProps) {const { t, i18n } = useTranslation("openhands");const headerKey = getAcpErrorHeaderKey(code);@@ -97,13 +100,24 @@ export function ErrorMessageBanner({)}data-testid="error-message-banner">- <CircleX- aria-hidden- className="h-4 w-4 shrink-0"- strokeWidth={2}- style={{ color: OH_STATUS_ERROR_COLOR }}- data-testid="error-message-banner-icon"- />+ {classification != null &&+ classification.kind !== "internal" &&+ classification.kind !== "unknown" ? (+ <CircleAlert+ aria-hidden+ className="h-4 w-4 shrink-0 text-[var(--oh-warning)]"+ strokeWidth={2}+ data-testid="warning-message-banner-icon"+ />+ ) : (+ <CircleX+ aria-hidden+ className="h-4 w-4 shrink-0"+ strokeWidth={2}+ style={{ color: OH_STATUS_ERROR_COLOR }}+ data-testid="error-message-banner-icon"+ />+ )}<div className="min-w-0 flex-1">{headerKey && (<divdiff --git a/src/components/shared/buttons/conversation-confirmation-buttons.tsx b/src/components/shared/buttons/conversation-confirmation-buttons.tsxindex 278bca4580..860985863d 100644--- a/src/components/shared/buttons/conversation-confirmation-buttons.tsx+++ b/src/components/shared/buttons/conversation-confirmation-buttons.tsx@@ -93,7 +93,8 @@ export function ConversationConfirmationButtons() {if (curAgentState !== AgentState.AWAITING_USER_CONFIRMATION ||!awaitingAction ||- submittedEventIds.includes(awaitingAction.id ?? "")+ (awaitingAction.id !== undefined &&+ submittedEventIds.includes(awaitingAction.id))) {return null;}diff --git a/src/contexts/conversation-websocket-context.tsx b/src/contexts/conversation-websocket-context.tsxindex 890b4648fd..bae553c18e 100644--- a/src/contexts/conversation-websocket-context.tsx+++ b/src/contexts/conversation-websocket-context.tsx@@ -535,15 +535,22 @@ export function ConversationWebSocketProvider({const errorEvent = event as| ConversationErrorEvent| ServerErrorEvent;+ const classification =+ "classification" in errorEvent ? errorEvent.classification : null;trackError({- message: errorEvent.detail,source: "conversation",metadata: {eventId: errorEvent.id,errorCode: errorEvent.code,},+ classification,});- setErrorMessage(errorEvent.detail, "conversation", errorEvent.code);+ setErrorMessage(+ errorEvent.detail,+ "conversation",+ errorEvent.code,+ classification,+ );} else {handleNonErrorEvent();}@@ -552,13 +559,13 @@ export function ConversationWebSocketProvider({// them for analytics but keep them out of the banner above the chat box.if (isAgentErrorEvent(event)) {trackError({- message: event.error,source: "agent",metadata: {eventId: event.id,toolName: event.tool_name,toolCallId: event.tool_call_id,},+ classification: event.classification,});}@@ -753,15 +760,22 @@ export function ConversationWebSocketProvider({const errorEvent = event as| ConversationErrorEvent| ServerErrorEvent;+ const classification =+ "classification" in errorEvent ? errorEvent.classification : null;trackError({- message: errorEvent.detail,source: "planning_conversation",metadata: {eventId: errorEvent.id,errorCode: errorEvent.code,},+ classification,});- setErrorMessage(errorEvent.detail, "conversation", errorEvent.code);+ setErrorMessage(+ errorEvent.detail,+ "conversation",+ errorEvent.code,+ classification,+ );} else {handleNonErrorEvent();}@@ -770,13 +784,13 @@ export function ConversationWebSocketProvider({// them for analytics but keep them out of the banner above the chat box.if (isAgentErrorEvent(event)) {trackError({- message: event.error,source: "planning_agent",metadata: {eventId: event.id,toolName: event.tool_name,toolCallId: event.tool_call_id,},+ classification: event.classification,});}diff --git a/src/stores/error-message-store.ts b/src/stores/error-message-store.tsindex 8235846f2a..0ea4181b6c 100644--- a/src/stores/error-message-store.ts+++ b/src/stores/error-message-store.ts@@ -1,4 +1,5 @@import { create } from "zustand";+import type { ErrorClassification } from "@openhands/typescript-client";/*** "connection" errors auto-clear once connectivity recovers; "conversation"@@ -16,6 +17,7 @@ interface ErrorMessageState {* null for errors that carry no code (connection errors, plain strings).*/errorCode: string | null;+ errorClassification: ErrorClassification | null;}interface ErrorMessageActions {@@ -23,6 +25,7 @@ interface ErrorMessageActions {message: string,type?: ErrorMessageType,code?: string | null,+ classification?: ErrorClassification | null,) => void;removeErrorMessage: () => void;/** Clears the error only when it is a transient connection error. */@@ -35,6 +38,7 @@ const initialState: ErrorMessageState = {errorMessage: null,errorType: null,errorCode: null,+ errorClassification: null,};export const useErrorMessageStore = create<ErrorMessageStore>((set) => ({@@ -44,11 +48,13 @@ export const useErrorMessageStore = create<ErrorMessageStore>((set) => ({message: string,type: ErrorMessageType = "conversation",code: string | null = null,+ classification: ErrorClassification | null = null,) =>set(() => ({errorMessage: message,errorType: type,errorCode: code,+ errorClassification: classification,})),removeErrorMessage: () => set(() => ({ ...initialState })),diff --git a/src/utils/error-handler.ts b/src/utils/error-handler.tsindex 2d7b863cb9..d95304246d 100644--- a/src/utils/error-handler.ts+++ b/src/utils/error-handler.ts@@ -1,15 +1,47 @@-import { trackException } from "#/services/telemetry";+import { trackEvent } from "#/services/telemetry";+import type { ErrorClassification } from "@openhands/typescript-client";interface ErrorDetails {- message: string;source?: string;metadata?: Record<string, unknown>;+ classification?: ErrorClassification | null;}-export function trackError({ message, source, metadata = {} }: ErrorDetails) {- const error = new Error(message);- void trackException(error, {+const RESERVED_ERROR_KEYS = new Set([+ "error_source",+ "error_kind",+ "error_id",+ "error_telemetry",+]);++export function trackError({+ source,+ metadata = {},+ classification,+}: ErrorDetails) {+ // Reserved outcome fields are derived from `source`/`classification` and+ // must not be overridable through arbitrary caller metadata.+ const extra = Object.fromEntries(+ Object.entries(metadata).filter(([key]) => !RESERVED_ERROR_KEYS.has(key)),+ );+ const kind = classification?.kind || "unknown";+ // Promote a caller-provided `eventId` to the reserved `error_id` dimension as+ // a fallback so unclassified errors stay correlatable with server-side logs.+ const promotedEventId =+ typeof metadata.eventId === "string" ? metadata.eventId : undefined;+ if (promotedEventId != null) {+ delete extra.eventId;+ }+ const errorId = classification?.error_id ?? promotedEventId;++ void trackEvent("error_outcome", {+ ...extra,+ current_url: window.location.href,error_source: source || "unknown",- ...metadata,+ error_kind: kind,+ // Keep diagnostic errors correlatable without capturing raw messages.+ ...(errorId != null ? { error_id: errorId } : {}),+ error_telemetry:+ kind === "internal" || kind === "unknown" ? "diagnostic" : "outcome",});}diff --git a/src/utils/transcript-export/load-complete-events.ts b/src/utils/transcript-export/load-complete-events.tsindex 06fdc1650f..185dc97aa9 100644--- a/src/utils/transcript-export/load-complete-events.ts+++ b/src/utils/transcript-export/load-complete-events.ts@@ -37,6 +37,10 @@ export const loadCompleteTranscriptEvents = async (): Promise<OpenHandsEvent[]> => {const persistedDescending: OpenHandsEvent[] = [];const fetchedEventIds = new Set<string>();+ // Id-less events can't be de-duplicated by id, so keep a separate ordered+ // list and a count so they survive the merge and count toward completeness.+ const idLessEvents: OpenHandsEvent[] = [];+ let idLessPersisted = 0;const seenPageIds = new Set<string>();let oldestTimestamp: string | undefined;let pageId: string | undefined;@@ -63,7 +67,11 @@ export const loadCompleteTranscriptEvents = async (let addedEvent = false;page.items.forEach((event) => {if (!fetchedEventIds.has(event.id ?? "")) {- if (event.id) fetchedEventIds.add(event.id);+ if (event.id) {+ fetchedEventIds.add(event.id);+ } else {+ idLessPersisted += 1;+ }addedEvent = true;}if (@@ -120,22 +128,34 @@ export const loadCompleteTranscriptEvents = async (.slice().reverse().forEach((event) => {- if (!eventsById.has(event.id ?? ""))- if (event.id) eventsById.set(event.id, event);+ if (event.id !== undefined) {+ if (!eventsById.has(event.id)) {+ eventsById.set(event.id, event);+ }+ return;+ }+ idLessEvents.push(event);});loadedEvents.forEach((event) => {- if (!eventsById.has(event.id ?? ""))- if (event.id) eventsById.set(event.id, event);+ if (event.id !== undefined) {+ if (!eventsById.has(event.id)) {+ eventsById.set(event.id, event);+ }+ return;+ }+ idLessEvents.push(event);});// Array.prototype.sort is stable, so equal-timestamp events keep the causal// order returned by the server/store rather than being reordered by id.- const completeEvents = [...eventsById.values()].sort(compareEventTimestamps);+ const completeEvents = [...idLessEvents, ...eventsById.values()].sort(+ compareEventTimestamps,+ );if (expectedEventCount !== undefined &&- fetchedEventIds.size < expectedEventCount+ fetchedEventIds.size + idLessPersisted < expectedEventCount) {throw new Error(- `Transcript history is incomplete: expected ${expectedEventCount} persisted events, received ${fetchedEventIds.size}.`,+ `Transcript history is incomplete: expected ${expectedEventCount} persisted events, received ${fetchedEventIds.size + idLessPersisted}.`,);}return completeEvents;diff --git a/tests/e2e/mock-llm/automations/mock-llm-automation.spec.ts b/tests/e2e/mock-llm/automations/mock-llm-automation.spec.tsindex 86bab27cd9..8a274e8ce9 100644--- a/tests/e2e/mock-llm/automations/mock-llm-automation.spec.ts+++ b/tests/e2e/mock-llm/automations/mock-llm-automation.spec.ts@@ -87,6 +87,29 @@ async function listAutomations();}+/**+ * Wait for a newly-created automation to become visible through the list API.+ * Creation and listing are separate backend operations, so the list can lag+ * briefly after the create request succeeds.+ */+async function waitForAutomation(+ request: import("@playwright/test").APIRequestContext,+ name: string,+ timeoutMs = 30_000,+) {+ const deadline = Date.now() + timeoutMs;+ while (Date.now() < deadline) {+ const data = await listAutomations(request, 1);+ const automations = data.automations ?? data.items ?? [];+ const automation = automations.find(+ (candidate: { name: string }) => candidate.name === name,+ );+ if (automation) return automation;+ await new Promise((resolve) => setTimeout(resolve, 1_000));+ }+ throw new Error(`Automation "${name}" was not visible after ${timeoutMs}ms`);+}+/*** List runs for a specific automation via the real automation backend.*/@@ -155,6 +178,20 @@ async function deleteAutomation();}+/** Remove stale automations with the test's fixed name before a retry. */+async function deleteAutomationsByName(+ request: import("@playwright/test").APIRequestContext,+ name: string,+) {+ const data = await listAutomations(request);+ const automations = data.automations ?? data.items ?? [];+ for (const automation of automations.filter(+ (candidate: { name: string }) => candidate.name === name,+ )) {+ await deleteAutomation(request, automation.id);+ }+}+test.describe.configure({ mode: "serial" });test.describe("mock-LLM automation lifecycle", () => {@@ -211,6 +248,10 @@ test.describe("mock-LLM automation lifecycle", () => {page,request,}) => {+ // Retries reuse the same Docker backend, so remove any automation left by+ // an earlier attempt before creating the fixed-name test resource.+ await deleteAutomationsByName(request, AUTOMATION_NAME);+// Ensure the mock LLM profile is configured via the Settings UIawait ensureMockLLMProfile(page);@@ -234,7 +275,7 @@ test.describe("mock-LLM automation lifecycle", () => {// startup in the uvx/bin dev paths; transient connect/reset/5xx// failures should not abort the create (observed flake β assert then// sees 0 automations).- `curl -s -X POST '${AUTOMATION_API_BASE}/preset/prompt'`,+ `curl --fail-with-body -sS -X POST '${AUTOMATION_API_BASE}/preset/prompt'`,`--retry 3 --retry-connrefused --retry-delay 1`,`-H 'Content-Type: application/json'`,authHeader,@@ -251,7 +292,7 @@ test.describe("mock-LLM automation lifecycle", () => {const dispatchCmd = [`AID=$(python3 -c "import json; print(json.load(open('/tmp/auto_result.json'))['id'])")`,- `&& curl -s -X POST "${AUTOMATION_API_BASE}/$AID/dispatch"`,+ `&& curl --fail-with-body -sS -X POST "${AUTOMATION_API_BASE}/$AID/dispatch"`,authHeader,`-H 'Content-Type: application/json'`,`-w '\\nHTTP_CODE:%{http_code}\\n'`,@@ -375,17 +416,7 @@ test.describe("mock-LLM automation lifecycle", () => {// ββ Verify: automation was created in the real automation backend ββawait test.step("verify automation was created", async () => {- const data = await listAutomations(request);- const automations = data.automations ?? data.items ?? [];- expect(- automations.length,- `Expected at least 1 automation, got: ${JSON.stringify(data).slice(0, 500)}`,- ).toBeGreaterThanOrEqual(1);-- const created = automations.find(- (a: { name: string }) => a.name === AUTOMATION_NAME,- );- expect(created, `Automation "${AUTOMATION_NAME}" not found`).toBeTruthy();+ const created = await waitForAutomation(request, AUTOMATION_NAME);automationIds.add(created.id);expect(created.trigger?.schedule).toBe(CRON_SCHEDULE);expect(created.enabled).toBe(true);@@ -394,12 +425,7 @@ test.describe("mock-LLM automation lifecycle", () => {// ββ Verify: run completed successfully with a conversation link ββawait test.step("verify run completed with conversation link", async () => {- const data = await listAutomations(request);- const automations = data.automations ?? data.items ?? [];- const automation = automations.find(- (a: { name: string }) => a.name === AUTOMATION_NAME,- );- expect(automation, "Automation should exist for run check").toBeTruthy();+ const automation = await waitForAutomation(request, AUTOMATION_NAME);automationIds.add(automation.id);// Wait for the run to reach COMPLETED. The trajectory includes extra@@ -450,7 +476,7 @@ test.describe("mock-LLM automation lifecycle", () => {: Array.isArray(msg.content)? (msg.content as Array<{ type?: string; text?: string }>).filter((c) => c.type === "text" || typeof c === "string")- .map((c) => (typeof c === "string" ? c : c.text ?? ""))+ .map((c) => (typeof c === "string" ? c : (c.text ?? ""))).join(""): "";if (text.includes("<RUNTIME_SERVICES>")) return text;@@ -523,7 +549,9 @@ test.describe("mock-LLM automation lifecycle", () => {// The ConfigurationSection renders schedule_human (e.g. "Every day at 9:00 AM")// or falls back to the raw cron expression.await test.step("verify cron schedule displayed on detail page", async () => {- await expect(page.getByText(CRON_SCHEDULE)).toBeVisible({ timeout: 10_000 });+ await expect(page.getByText(CRON_SCHEDULE)).toBeVisible({+ timeout: 10_000,+ });});await test.step("verify run shows COMPLETED with conversation link", async () => {@@ -564,5 +592,4 @@ test.describe("mock-LLM automation lifecycle", () => {}});});-});diff --git a/tests/e2e/mock-llm/mcp/mock-llm-mcp-slack-credentials.spec.ts b/tests/e2e/mock-llm/mcp/mock-llm-mcp-slack-credentials.spec.tsindex 611fae8473..bc0b6e571a 100644--- a/tests/e2e/mock-llm/mcp/mock-llm-mcp-slack-credentials.spec.ts+++ b/tests/e2e/mock-llm/mcp/mock-llm-mcp-slack-credentials.spec.ts@@ -119,7 +119,7 @@ async function openSlackInstallModal(page: Page) {await page.goto("/mcp", { waitUntil: "domcontentloaded" });await dismissAnalyticsModal(page);await expect(page.getByTestId("mcp-marketplace-grid")).toBeVisible({- timeout: 10_000,+ timeout: 30_000,});await page.getByTestId("mcp-marketplace-card-slack").click();const modal = page.getByTestId("mcp-install-modal");diff --git a/tests/e2e/mock-llm/settings/mock-llm-profile-management.spec.ts b/tests/e2e/mock-llm/settings/mock-llm-profile-management.spec.tsindex 823221c7a7..24add6cea8 100644--- a/tests/e2e/mock-llm/settings/mock-llm-profile-management.spec.ts+++ b/tests/e2e/mock-llm/settings/mock-llm-profile-management.spec.ts@@ -40,6 +40,7 @@ import {setChatInput,waitForPath,createProfileViaUI,+ editProfileViaUI,deleteProfileIfExists,activateProfileViaUI,ensureMockLLMAgentProfile,@@ -414,12 +415,22 @@ test.describe("OpenHands provider hidden base_url preservation", () => {await dismissAnalyticsModal(page);await waitForTestId(page, "add-llm-profile");- await deleteProfileIfExists(page, OPENHANDS_PROFILE);- await createProfileViaUI(page, {- profileName: OPENHANDS_PROFILE,- model: OPENHANDS_MODEL,- baseUrl: CUSTOM_BASE_URL,- });+ const existingProfile = page+ .getByTestId("profile-row")+ .filter({ has: page.locator(`span[title="${OPENHANDS_PROFILE}"]`) });+ if ((await existingProfile.count()) > 0) {+ await editProfileViaUI(page, {+ profileName: OPENHANDS_PROFILE,+ model: OPENHANDS_MODEL,+ baseUrl: CUSTOM_BASE_URL,+ });+ } else {+ await createProfileViaUI(page, {+ profileName: OPENHANDS_PROFILE,+ model: OPENHANDS_MODEL,+ baseUrl: CUSTOM_BASE_URL,+ });+ }await test.step("open profile in edit mode", async () => {const profileRows = page.getByTestId("profile-row");