Skip to content

πŸ™Œ OpenHands: AI-Driven Development

feat(canvas): present structured error outcomes (#16231)

Graham Neubig0d15c5e79e91
diff --git a/__tests__/components/chat/error-message-banner.test.tsx b/__tests__/components/chat/error-message-banner.test.tsx
index 1535d793dd..ede6142475 100644
--- a/__tests__/components/chat/error-message-banner.test.tsx
+++ b/__tests__/components/chat/error-message-banner.test.tsx
@@ -54,6 +54,57 @@ describe("ErrorMessageBanner", () => {
expect(icon).toHaveStyle({ color: "var(--oh-status-error)" });
});
+ it("uses a warning icon for non-internal outcomes", () => {
+ render(
+ <ErrorMessageBanner
+ message="Incorrect API key"
+ classification={{
+ kind: "auth",
+ retryable: false,
+ user_action: "settings",
+ }}
+ />,
+ );
+
+ expect(screen.getByTestId("warning-message-banner-icon")).toHaveClass(
+ "text-[var(--oh-warning)]",
+ );
+ });
+
+ it("uses the error icon for internal outcomes", () => {
+ render(
+ <ErrorMessageBanner
+ message="Something went wrong"
+ classification={{
+ kind: "internal",
+ retryable: false,
+ user_action: "none",
+ }}
+ />,
+ );
+
+ expect(screen.getByTestId("error-message-banner-icon")).toHaveStyle({
+ color: "var(--oh-status-error)",
+ });
+ });
+
+ it("uses the error icon for unknown (diagnostic) outcomes", () => {
+ render(
+ <ErrorMessageBanner
+ message="Something went wrong"
+ classification={{
+ kind: "unknown",
+ retryable: false,
+ user_action: "none",
+ }}
+ />,
+ );
+
+ expect(screen.getByTestId("error-message-banner-icon")).toHaveStyle({
+ color: "var(--oh-status-error)",
+ });
+ });
+
it("uses greyscale theme tokens instead of red error styling", () => {
render(<ErrorMessageBanner message="Something went wrong" />);
diff --git a/__tests__/contexts/conversation-websocket-context.test.tsx b/__tests__/contexts/conversation-websocket-context.test.tsx
index 1147b8dd10..05dd3c113f 100644
--- a/__tests__/contexts/conversation-websocket-context.test.tsx
+++ b/__tests__/contexts/conversation-websocket-context.test.tsx
@@ -27,12 +27,17 @@ type CapturedWebSocketOptions = {
const wsCapture = vi.hoisted(() => ({
mainOnMessage: null as null | ((event: { data: string }) => void),
mainOptions: null as CapturedWebSocketOptions | null,
+ planningOnMessage: null as null | ((event: { data: string }) => void),
calls: [] as Array<{
url: string;
options?: CapturedWebSocketOptions;
}>,
}));
+const errorHandlerMocks = vi.hoisted(() => ({
+ trackError: vi.fn(),
+}));
+
// Keep the units under test real (the provider, `useConversationHistory`, the
// event store). Only the network is stubbed: the WebSocket transport and the
// REST service the history query depends on.
@@ -50,12 +55,23 @@ vi.mock("#/hooks/use-websocket", () => ({
wsCapture.mainOnMessage = options.onMessage;
wsCapture.mainOptions = options;
}
+ if (
+ url &&
+ options?.onMessage &&
+ options.queryParams &&
+ "resend_all" in options.queryParams
+ ) {
+ wsCapture.planningOnMessage = options.onMessage;
+ }
return { socket: null, reconnect: vi.fn() };
}),
}));
vi.mock("#/hooks/query/use-user-conversation", () => ({
useUserConversation: vi.fn(),
}));
+vi.mock("#/utils/error-handler", () => ({
+ trackError: errorHandlerMocks.trackError,
+}));
const AGENT_REPLY_ID = "evt-agent-reply";
@@ -91,6 +107,7 @@ describe("ConversationWebSocketProvider β€” conversation-scoped event store", ()
beforeEach(() => {
wsCapture.mainOnMessage = null;
wsCapture.mainOptions = null;
+ wsCapture.planningOnMessage = null;
wsCapture.calls.length = 0;
window.localStorage.clear();
queryClient = new QueryClient({
@@ -356,13 +373,42 @@ describe("ConversationWebSocketProvider β€” conversation-scoped event store", ()
},
});
- const makeConversationError = (id: string, detail: string) => ({
+ const makeConversationError = (
+ id: string,
+ detail: string,
+ classification?: {
+ kind: "auth";
+ retryable: boolean;
+ user_action: "settings";
+ },
+ ) => ({
id,
timestamp: new Date().toISOString(),
source: "environment",
kind: "ConversationErrorEvent",
detail,
code: "SomeError",
+ ...(classification ? { classification } : {}),
+ });
+
+ const makeAgentError = (
+ id: string,
+ classification?: {
+ kind: "auth";
+ retryable: boolean;
+ user_action: "settings";
+ },
+ ) => ({
+ id,
+ timestamp: new Date().toISOString(),
+ source: "agent",
+ message_id: `msg-${id}`,
+ message_seq: 1,
+ error: "Agent failed",
+ error_type: "AgentError",
+ tool_name: "generic",
+ tool_call_id: `call-${id}`,
+ ...(classification ? { classification } : {}),
});
const renderCaptured = async () => {
@@ -421,6 +467,115 @@ describe("ConversationWebSocketProvider β€” conversation-scoped event store", ()
deliver(errorEvent);
expect(useErrorMessageStore.getState().errorMessage).toBeNull();
});
+
+ it("forwards error classifications to the banner store and telemetry", async () => {
+ await renderCaptured();
+ const classification = {
+ kind: "auth" as const,
+ retryable: false,
+ user_action: "settings" as const,
+ };
+
+ deliver(
+ makeConversationError(
+ "conv-error-2",
+ "Authentication failed",
+ classification,
+ ),
+ );
+
+ expect(useErrorMessageStore.getState().errorClassification).toEqual(
+ classification,
+ );
+ expect(errorHandlerMocks.trackError).toHaveBeenCalledWith({
+ source: "conversation",
+ metadata: {
+ eventId: "conv-error-2",
+ errorCode: "SomeError",
+ },
+ classification,
+ });
+ });
+
+ it("forwards AgentErrorEvent classifications to telemetry (main agent)", async () => {
+ await renderCaptured();
+ const classification = {
+ kind: "auth" as const,
+ retryable: false,
+ user_action: "settings" as const,
+ };
+
+ deliver(makeAgentError("agent-err-1", classification));
+
+ expect(errorHandlerMocks.trackError).toHaveBeenCalledWith({
+ source: "agent",
+ metadata: {
+ eventId: "agent-err-1",
+ toolName: "generic",
+ toolCallId: "call-agent-err-1",
+ },
+ classification,
+ });
+ });
+
+ it("forwards AgentErrorEvent classifications to telemetry (planning agent)", async () => {
+ const planningConversation: AppConversation = {
+ id: "planning-err",
+ created_by_user_id: null,
+ selected_repository: null,
+ selected_branch: null,
+ git_provider: null,
+ title: "Planner",
+ trigger: null,
+ pr_number: [],
+ llm_model: null,
+ metrics: null,
+ created_at: "2026-07-28T00:00:00Z",
+ updated_at: "2026-07-28T00:00:00Z",
+ execution_status: null,
+ conversation_url:
+ "http://planner.example/api/conversations/planning-err",
+ session_api_key: null,
+ sandbox_id: null,
+ sub_conversation_ids: [],
+ };
+ const classification = {
+ kind: "auth" as const,
+ retryable: true,
+ user_action: "settings" as const,
+ };
+
+ render(
+ <QueryClientProvider client={queryClient}>
+ <ConversationWebSocketProvider
+ conversationId="conv-err"
+ conversationUrl="http://main.example/api/conversations/conv-err"
+ subConversationIds={[planningConversation.id]}
+ subConversations={[planningConversation]}
+ >
+ <div />
+ </ConversationWebSocketProvider>
+ </QueryClientProvider>,
+ );
+ // Wait for the planning sub-conversation WebSocket to be established.
+ await waitFor(() => expect(wsCapture.planningOnMessage).not.toBeNull());
+
+ act(() => {
+ wsCapture.planningOnMessage!({
+ data: JSON.stringify(makeAgentError("agent-err-2", classification)),
+ });
+ });
+
+ expect(errorHandlerMocks.trackError).toHaveBeenCalledWith({
+ source: "planning_agent",
+ metadata: {
+ eventId: "agent-err-2",
+ toolName: "generic",
+ toolCallId: "call-agent-err-2",
+ },
+ classification,
+ });
+ });
});
it("clears the previous conversation's events when switching conversations", async () => {
diff --git a/__tests__/stores/error-message-store.test.ts b/__tests__/stores/error-message-store.test.ts
index f768424275..6b19812747 100644
--- a/__tests__/stores/error-message-store.test.ts
+++ b/__tests__/stores/error-message-store.test.ts
@@ -9,6 +9,7 @@ describe("error message store", () => {
errorMessage: null,
errorType: null,
errorCode: null,
+ errorClassification: null,
});
});
@@ -58,6 +59,25 @@ describe("error message store", () => {
expect(getState().errorCode).toBe("ACPAuthRequired");
});
+ it("stores and clears structured error classifications", () => {
+ const classification = {
+ kind: "auth" as const,
+ retryable: false,
+ user_action: "settings" as const,
+ };
+
+ getState().setErrorMessage(
+ "auth failed",
+ "conversation",
+ null,
+ classification,
+ );
+ expect(getState().errorClassification).toEqual(classification);
+
+ getState().removeErrorMessage();
+ expect(getState().errorClassification).toBeNull();
+ });
+
it("removeErrorMessage clears the error code too", () => {
getState().setErrorMessage("auth failed", "conversation", "ACPAuthRequired");
getState().removeErrorMessage();
diff --git a/__tests__/utils/error-handler.test.ts b/__tests__/utils/error-handler.test.ts
index 3aba0ef374..bbc1b1d167 100644
--- a/__tests__/utils/error-handler.test.ts
+++ b/__tests__/utils/error-handler.test.ts
@@ -1,12 +1,15 @@
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
import { trackError } from "#/utils/error-handler";
-import { trackException } from "#/services/telemetry";
+import { trackEvent } from "#/services/telemetry";
vi.mock("#/services/telemetry", () => ({
- trackException: vi.fn(),
+ trackEvent: vi.fn(),
}));
describe("Error Handler", () => {
+ beforeEach(() => {
+ window.history.replaceState({}, "", "/test-error");
+ });
beforeEach(() => {
vi.clearAllMocks();
});
@@ -18,40 +21,122 @@ describe("Error Handler", () => {
describe("trackError", () => {
it("should send error to PostHog with basic info", () => {
const error = {
- message: "Test error",
source: "test",
};
trackError(error);
- expect(trackException).toHaveBeenCalledWith(
- new Error("Test error"),
- {
- error_source: "test",
- },
- );
+ expect(trackEvent).toHaveBeenCalledWith("error_outcome", {
+ current_url: window.location.href,
+ error_source: "test",
+ error_kind: "unknown",
+ error_telemetry: "diagnostic",
+ });
});
- it("should include additional metadata in PostHog event", () => {
+ it("merges metadata while reserving error outcome fields", () => {
const error = {
- message: "Test error",
source: "test",
metadata: {
extra: "info",
details: { foo: "bar" },
+ error_kind: "spoofed",
+ error_telemetry: "outcome",
+ error_id: "spoofed",
},
};
trackError(error);
- expect(trackException).toHaveBeenCalledWith(
- new Error("Test error"),
- {
- error_source: "test",
- extra: "info",
- details: { foo: "bar" },
+ expect(trackEvent).toHaveBeenCalledWith("error_outcome", {
+ current_url: window.location.href,
+ error_source: "test",
+ error_kind: "unknown",
+ error_telemetry: "diagnostic",
+ extra: "info",
+ details: { foo: "bar" },
+ });
+ });
+
+ it("keeps a classified error ID without recording the message", () => {
+ trackError({
+ source: "agent",
+ classification: {
+ kind: "internal",
+ retryable: false,
+ user_action: "none",
+ error_id: "error-123",
},
+ });
+
+ expect(trackEvent).toHaveBeenCalledWith("error_outcome", {
+ current_url: window.location.href,
+ error_source: "agent",
+ error_kind: "internal",
+ error_id: "error-123",
+ error_telemetry: "diagnostic",
+ });
+ });
+
+ it("promotes a string metadata eventId as the error_id fallback", () => {
+ trackError({
+ source: "chat",
+ metadata: { msgId: "m-1", eventId: "evt-42", extra: "info" },
+ });
+
+ expect(trackEvent).toHaveBeenCalledWith("error_outcome", {
+ current_url: window.location.href,
+ error_source: "chat",
+ error_kind: "unknown",
+ error_id: "evt-42",
+ error_telemetry: "diagnostic",
+ msgId: "m-1",
+ extra: "info",
+ });
+ });
+
+ it("lets a classification error_id win over a metadata eventId", () => {
+ trackError({
+ source: "agent",
+ metadata: { msgId: "m-1", eventId: "evt-42" },
+ classification: {
+ kind: "auth",
+ retryable: false,
+ user_action: "settings",
+ error_id: "cls-7",
+ },
+ });
+
+ expect(trackEvent).toHaveBeenCalledWith(
+ "error_outcome",
+ expect.objectContaining({
+ current_url: window.location.href,
+ error_id: "cls-7",
+ error_telemetry: "outcome",
+ }),
);
+ expect(trackEvent).not.toHaveBeenCalledWith(
+ "error_outcome",
+ expect.objectContaining({ eventId: "evt-42" }),
+ );
+ });
+
+ it("records non-internal classifications as outcomes", () => {
+ trackError({
+ source: "agent",
+ classification: {
+ kind: "auth",
+ retryable: false,
+ user_action: "settings",
+ },
+ });
+
+ expect(trackEvent).toHaveBeenCalledWith("error_outcome", {
+ current_url: window.location.href,
+ error_source: "agent",
+ error_kind: "auth",
+ error_telemetry: "outcome",
+ });
});
});
});
diff --git a/src/components/features/chat/chat-interface.tsx b/src/components/features/chat/chat-interface.tsx
index 72c80ba4ec..5521222962 100644
--- a/src/components/features/chat/chat-interface.tsx
+++ b/src/components/features/chat/chat-interface.tsx
@@ -64,8 +64,13 @@ function getEntryPoint(
export function ChatInterface() {
const { trackInitialQuerySubmitted, trackUserMessageSent } = useTracking();
const { setMessageToSend } = useConversationStore();
- const { errorMessage, errorCode, removeErrorMessage, setErrorMessage } =
- useErrorMessageStore();
+ const {
+ errorMessage,
+ errorCode,
+ errorClassification,
+ removeErrorMessage,
+ setErrorMessage,
+ } = useErrorMessageStore();
const navigate = useNavigate();
const { isTask, taskStatus, taskDetail } = useTaskPolling();
// Hide empty-state chrome for the entire `/conversations/task-{uuid}` route,
@@ -570,6 +575,7 @@ export function ChatInterface() {
<ErrorMessageBanner
message={errorMessage}
code={errorCode}
+ classification={errorClassification}
onDismiss={removeErrorMessage}
onRetry={
errorMessage === SERVER_CONNECTION_ERROR_MESSAGE
diff --git a/src/components/features/chat/error-message-banner.tsx b/src/components/features/chat/error-message-banner.tsx
index 888f0fa116..cff1972000 100644
--- a/src/components/features/chat/error-message-banner.tsx
+++ b/src/components/features/chat/error-message-banner.tsx
@@ -1,11 +1,12 @@
import React from "react";
import { Trans, useTranslation } from "react-i18next";
-import { Check, CircleX, Copy, X } from "lucide-react";
+import { Check, CircleAlert, CircleX, Copy, X } from "lucide-react";
import { OH_STATUS_ERROR_COLOR } from "#/constants/status-colors";
import { I18nKey } from "#/i18n/declaration";
import { displayErrorToast } from "#/utils/custom-toast-handlers";
import { getAcpErrorHeaderKey } from "#/utils/acp-error-codes";
import { cn } from "#/utils/utils";
+import type { ErrorClassification } from "@openhands/typescript-client";
interface ErrorMessageBannerProps {
message: string;
@@ -15,6 +16,7 @@ interface ErrorMessageBannerProps {
onRetry?: () => void;
/** Recovery action (e.g. re-authenticate) shown for credential failures. */
onReauth?: () => void;
+ classification?: ErrorClassification | null;
}
const DEFAULT_MAX_COLLAPSED_CHARS = 220;
@@ -25,6 +27,7 @@ export function ErrorMessageBanner({
onDismiss,
onRetry,
onReauth,
+ classification,
}: ErrorMessageBannerProps) {
const { t, i18n } = useTranslation("openhands");
const headerKey = getAcpErrorHeaderKey(code);
@@ -97,13 +100,24 @@ export function ErrorMessageBanner({
)}
data-testid="error-message-banner"
>
- <CircleX
- aria-hidden
- className="h-4 w-4 shrink-0"
- strokeWidth={2}
- style={{ color: OH_STATUS_ERROR_COLOR }}
- data-testid="error-message-banner-icon"
- />
+ {classification != null &&
+ classification.kind !== "internal" &&
+ classification.kind !== "unknown" ? (
+ <CircleAlert
+ aria-hidden
+ className="h-4 w-4 shrink-0 text-[var(--oh-warning)]"
+ strokeWidth={2}
+ data-testid="warning-message-banner-icon"
+ />
+ ) : (
+ <CircleX
+ aria-hidden
+ className="h-4 w-4 shrink-0"
+ strokeWidth={2}
+ style={{ color: OH_STATUS_ERROR_COLOR }}
+ data-testid="error-message-banner-icon"
+ />
+ )}
<div className="min-w-0 flex-1">
{headerKey && (
<div
diff --git a/src/components/shared/buttons/conversation-confirmation-buttons.tsx b/src/components/shared/buttons/conversation-confirmation-buttons.tsx
index 278bca4580..860985863d 100644
--- a/src/components/shared/buttons/conversation-confirmation-buttons.tsx
+++ b/src/components/shared/buttons/conversation-confirmation-buttons.tsx
@@ -93,7 +93,8 @@ export function ConversationConfirmationButtons() {
if (
curAgentState !== AgentState.AWAITING_USER_CONFIRMATION ||
!awaitingAction ||
- submittedEventIds.includes(awaitingAction.id ?? "")
+ (awaitingAction.id !== undefined &&
+ submittedEventIds.includes(awaitingAction.id))
) {
return null;
}
diff --git a/src/contexts/conversation-websocket-context.tsx b/src/contexts/conversation-websocket-context.tsx
index 890b4648fd..bae553c18e 100644
--- a/src/contexts/conversation-websocket-context.tsx
+++ b/src/contexts/conversation-websocket-context.tsx
@@ -535,15 +535,22 @@ export function ConversationWebSocketProvider({
const errorEvent = event as
| ConversationErrorEvent
| ServerErrorEvent;
+ const classification =
+ "classification" in errorEvent ? errorEvent.classification : null;
trackError({
- message: errorEvent.detail,
source: "conversation",
metadata: {
eventId: errorEvent.id,
errorCode: errorEvent.code,
},
+ classification,
});
- setErrorMessage(errorEvent.detail, "conversation", errorEvent.code);
+ setErrorMessage(
+ errorEvent.detail,
+ "conversation",
+ errorEvent.code,
+ classification,
+ );
} else {
handleNonErrorEvent();
}
@@ -552,13 +559,13 @@ export function ConversationWebSocketProvider({
// them for analytics but keep them out of the banner above the chat box.
if (isAgentErrorEvent(event)) {
trackError({
- message: event.error,
source: "agent",
metadata: {
eventId: event.id,
toolName: event.tool_name,
toolCallId: event.tool_call_id,
},
+ classification: event.classification,
});
}
@@ -753,15 +760,22 @@ export function ConversationWebSocketProvider({
const errorEvent = event as
| ConversationErrorEvent
| ServerErrorEvent;
+ const classification =
+ "classification" in errorEvent ? errorEvent.classification : null;
trackError({
- message: errorEvent.detail,
source: "planning_conversation",
metadata: {
eventId: errorEvent.id,
errorCode: errorEvent.code,
},
+ classification,
});
- setErrorMessage(errorEvent.detail, "conversation", errorEvent.code);
+ setErrorMessage(
+ errorEvent.detail,
+ "conversation",
+ errorEvent.code,
+ classification,
+ );
} else {
handleNonErrorEvent();
}
@@ -770,13 +784,13 @@ export function ConversationWebSocketProvider({
// them for analytics but keep them out of the banner above the chat box.
if (isAgentErrorEvent(event)) {
trackError({
- message: event.error,
source: "planning_agent",
metadata: {
eventId: event.id,
toolName: event.tool_name,
toolCallId: event.tool_call_id,
},
+ classification: event.classification,
});
}
diff --git a/src/stores/error-message-store.ts b/src/stores/error-message-store.ts
index 8235846f2a..0ea4181b6c 100644
--- a/src/stores/error-message-store.ts
+++ b/src/stores/error-message-store.ts
@@ -1,4 +1,5 @@
import { create } from "zustand";
+import type { ErrorClassification } from "@openhands/typescript-client";
/**
* "connection" errors auto-clear once connectivity recovers; "conversation"
@@ -16,6 +17,7 @@ interface ErrorMessageState {
* null for errors that carry no code (connection errors, plain strings).
*/
errorCode: string | null;
+ errorClassification: ErrorClassification | null;
}
interface ErrorMessageActions {
@@ -23,6 +25,7 @@ interface ErrorMessageActions {
message: string,
type?: ErrorMessageType,
code?: string | null,
+ classification?: ErrorClassification | null,
) => void;
removeErrorMessage: () => void;
/** Clears the error only when it is a transient connection error. */
@@ -35,6 +38,7 @@ const initialState: ErrorMessageState = {
errorMessage: null,
errorType: null,
errorCode: null,
+ errorClassification: null,
};
export const useErrorMessageStore = create<ErrorMessageStore>((set) => ({
@@ -44,11 +48,13 @@ export const useErrorMessageStore = create<ErrorMessageStore>((set) => ({
message: string,
type: ErrorMessageType = "conversation",
code: string | null = null,
+ classification: ErrorClassification | null = null,
) =>
set(() => ({
errorMessage: message,
errorType: type,
errorCode: code,
+ errorClassification: classification,
})),
removeErrorMessage: () => set(() => ({ ...initialState })),
diff --git a/src/utils/error-handler.ts b/src/utils/error-handler.ts
index 2d7b863cb9..d95304246d 100644
--- a/src/utils/error-handler.ts
+++ b/src/utils/error-handler.ts
@@ -1,15 +1,47 @@
-import { trackException } from "#/services/telemetry";
+import { trackEvent } from "#/services/telemetry";
+import type { ErrorClassification } from "@openhands/typescript-client";
interface ErrorDetails {
- message: string;
source?: string;
metadata?: Record<string, unknown>;
+ classification?: ErrorClassification | null;
}
-export function trackError({ message, source, metadata = {} }: ErrorDetails) {
- const error = new Error(message);
- void trackException(error, {
+const RESERVED_ERROR_KEYS = new Set([
+ "error_source",
+ "error_kind",
+ "error_id",
+ "error_telemetry",
+]);
+
+export function trackError({
+ source,
+ metadata = {},
+ classification,
+}: ErrorDetails) {
+ // Reserved outcome fields are derived from `source`/`classification` and
+ // must not be overridable through arbitrary caller metadata.
+ const extra = Object.fromEntries(
+ Object.entries(metadata).filter(([key]) => !RESERVED_ERROR_KEYS.has(key)),
+ );
+ const kind = classification?.kind || "unknown";
+ // Promote a caller-provided `eventId` to the reserved `error_id` dimension as
+ // a fallback so unclassified errors stay correlatable with server-side logs.
+ const promotedEventId =
+ typeof metadata.eventId === "string" ? metadata.eventId : undefined;
+ if (promotedEventId != null) {
+ delete extra.eventId;
+ }
+ const errorId = classification?.error_id ?? promotedEventId;
+
+ void trackEvent("error_outcome", {
+ ...extra,
+ current_url: window.location.href,
error_source: source || "unknown",
- ...metadata,
+ error_kind: kind,
+ // Keep diagnostic errors correlatable without capturing raw messages.
+ ...(errorId != null ? { error_id: errorId } : {}),
+ error_telemetry:
+ kind === "internal" || kind === "unknown" ? "diagnostic" : "outcome",
});
}
diff --git a/src/utils/transcript-export/load-complete-events.ts b/src/utils/transcript-export/load-complete-events.ts
index 06fdc1650f..185dc97aa9 100644
--- a/src/utils/transcript-export/load-complete-events.ts
+++ b/src/utils/transcript-export/load-complete-events.ts
@@ -37,6 +37,10 @@ export const loadCompleteTranscriptEvents = async (
): Promise<OpenHandsEvent[]> => {
const persistedDescending: OpenHandsEvent[] = [];
const fetchedEventIds = new Set<string>();
+ // Id-less events can't be de-duplicated by id, so keep a separate ordered
+ // list and a count so they survive the merge and count toward completeness.
+ const idLessEvents: OpenHandsEvent[] = [];
+ let idLessPersisted = 0;
const seenPageIds = new Set<string>();
let oldestTimestamp: string | undefined;
let pageId: string | undefined;
@@ -63,7 +67,11 @@ export const loadCompleteTranscriptEvents = async (
let addedEvent = false;
page.items.forEach((event) => {
if (!fetchedEventIds.has(event.id ?? "")) {
- if (event.id) fetchedEventIds.add(event.id);
+ if (event.id) {
+ fetchedEventIds.add(event.id);
+ } else {
+ idLessPersisted += 1;
+ }
addedEvent = true;
}
if (
@@ -120,22 +128,34 @@ export const loadCompleteTranscriptEvents = async (
.slice()
.reverse()
.forEach((event) => {
- if (!eventsById.has(event.id ?? ""))
- if (event.id) eventsById.set(event.id, event);
+ if (event.id !== undefined) {
+ if (!eventsById.has(event.id)) {
+ eventsById.set(event.id, event);
+ }
+ return;
+ }
+ idLessEvents.push(event);
});
loadedEvents.forEach((event) => {
- if (!eventsById.has(event.id ?? ""))
- if (event.id) eventsById.set(event.id, event);
+ if (event.id !== undefined) {
+ if (!eventsById.has(event.id)) {
+ eventsById.set(event.id, event);
+ }
+ return;
+ }
+ idLessEvents.push(event);
});
// Array.prototype.sort is stable, so equal-timestamp events keep the causal
// order returned by the server/store rather than being reordered by id.
- const completeEvents = [...eventsById.values()].sort(compareEventTimestamps);
+ const completeEvents = [...idLessEvents, ...eventsById.values()].sort(
+ compareEventTimestamps,
+ );
if (
expectedEventCount !== undefined &&
- fetchedEventIds.size < expectedEventCount
+ fetchedEventIds.size + idLessPersisted < expectedEventCount
) {
throw new Error(
- `Transcript history is incomplete: expected ${expectedEventCount} persisted events, received ${fetchedEventIds.size}.`,
+ `Transcript history is incomplete: expected ${expectedEventCount} persisted events, received ${fetchedEventIds.size + idLessPersisted}.`,
);
}
return completeEvents;
diff --git a/tests/e2e/mock-llm/automations/mock-llm-automation.spec.ts b/tests/e2e/mock-llm/automations/mock-llm-automation.spec.ts
index 86bab27cd9..8a274e8ce9 100644
--- a/tests/e2e/mock-llm/automations/mock-llm-automation.spec.ts
+++ b/tests/e2e/mock-llm/automations/mock-llm-automation.spec.ts
@@ -87,6 +87,29 @@ async function listAutomations(
);
}
+/**
+ * Wait for a newly-created automation to become visible through the list API.
+ * Creation and listing are separate backend operations, so the list can lag
+ * briefly after the create request succeeds.
+ */
+async function waitForAutomation(
+ request: import("@playwright/test").APIRequestContext,
+ name: string,
+ timeoutMs = 30_000,
+) {
+ const deadline = Date.now() + timeoutMs;
+ while (Date.now() < deadline) {
+ const data = await listAutomations(request, 1);
+ const automations = data.automations ?? data.items ?? [];
+ const automation = automations.find(
+ (candidate: { name: string }) => candidate.name === name,
+ );
+ if (automation) return automation;
+ await new Promise((resolve) => setTimeout(resolve, 1_000));
+ }
+ throw new Error(`Automation "${name}" was not visible after ${timeoutMs}ms`);
+}
+
/**
* List runs for a specific automation via the real automation backend.
*/
@@ -155,6 +178,20 @@ async function deleteAutomation(
);
}
+/** Remove stale automations with the test's fixed name before a retry. */
+async function deleteAutomationsByName(
+ request: import("@playwright/test").APIRequestContext,
+ name: string,
+) {
+ const data = await listAutomations(request);
+ const automations = data.automations ?? data.items ?? [];
+ for (const automation of automations.filter(
+ (candidate: { name: string }) => candidate.name === name,
+ )) {
+ await deleteAutomation(request, automation.id);
+ }
+}
+
test.describe.configure({ mode: "serial" });
test.describe("mock-LLM automation lifecycle", () => {
@@ -211,6 +248,10 @@ test.describe("mock-LLM automation lifecycle", () => {
page,
request,
}) => {
+ // Retries reuse the same Docker backend, so remove any automation left by
+ // an earlier attempt before creating the fixed-name test resource.
+ await deleteAutomationsByName(request, AUTOMATION_NAME);
+
// Ensure the mock LLM profile is configured via the Settings UI
await ensureMockLLMProfile(page);
@@ -234,7 +275,7 @@ test.describe("mock-LLM automation lifecycle", () => {
// startup in the uvx/bin dev paths; transient connect/reset/5xx
// failures should not abort the create (observed flake β†’ assert then
// sees 0 automations).
- `curl -s -X POST '${AUTOMATION_API_BASE}/preset/prompt'`,
+ `curl --fail-with-body -sS -X POST '${AUTOMATION_API_BASE}/preset/prompt'`,
`--retry 3 --retry-connrefused --retry-delay 1`,
`-H 'Content-Type: application/json'`,
authHeader,
@@ -251,7 +292,7 @@ test.describe("mock-LLM automation lifecycle", () => {
const dispatchCmd = [
`AID=$(python3 -c "import json; print(json.load(open('/tmp/auto_result.json'))['id'])")`,
- `&& curl -s -X POST "${AUTOMATION_API_BASE}/$AID/dispatch"`,
+ `&& curl --fail-with-body -sS -X POST "${AUTOMATION_API_BASE}/$AID/dispatch"`,
authHeader,
`-H 'Content-Type: application/json'`,
`-w '\\nHTTP_CODE:%{http_code}\\n'`,
@@ -375,17 +416,7 @@ test.describe("mock-LLM automation lifecycle", () => {
// ── Verify: automation was created in the real automation backend ──
await test.step("verify automation was created", async () => {
- const data = await listAutomations(request);
- const automations = data.automations ?? data.items ?? [];
- expect(
- automations.length,
- `Expected at least 1 automation, got: ${JSON.stringify(data).slice(0, 500)}`,
- ).toBeGreaterThanOrEqual(1);
-
- const created = automations.find(
- (a: { name: string }) => a.name === AUTOMATION_NAME,
- );
- expect(created, `Automation "${AUTOMATION_NAME}" not found`).toBeTruthy();
+ const created = await waitForAutomation(request, AUTOMATION_NAME);
automationIds.add(created.id);
expect(created.trigger?.schedule).toBe(CRON_SCHEDULE);
expect(created.enabled).toBe(true);
@@ -394,12 +425,7 @@ test.describe("mock-LLM automation lifecycle", () => {
// ── Verify: run completed successfully with a conversation link ──
await test.step("verify run completed with conversation link", async () => {
- const data = await listAutomations(request);
- const automations = data.automations ?? data.items ?? [];
- const automation = automations.find(
- (a: { name: string }) => a.name === AUTOMATION_NAME,
- );
- expect(automation, "Automation should exist for run check").toBeTruthy();
+ const automation = await waitForAutomation(request, AUTOMATION_NAME);
automationIds.add(automation.id);
// Wait for the run to reach COMPLETED. The trajectory includes extra
@@ -450,7 +476,7 @@ test.describe("mock-LLM automation lifecycle", () => {
: Array.isArray(msg.content)
? (msg.content as Array<{ type?: string; text?: string }>)
.filter((c) => c.type === "text" || typeof c === "string")
- .map((c) => (typeof c === "string" ? c : c.text ?? ""))
+ .map((c) => (typeof c === "string" ? c : (c.text ?? "")))
.join("")
: "";
if (text.includes("<RUNTIME_SERVICES>")) return text;
@@ -523,7 +549,9 @@ test.describe("mock-LLM automation lifecycle", () => {
// The ConfigurationSection renders schedule_human (e.g. "Every day at 9:00 AM")
// or falls back to the raw cron expression.
await test.step("verify cron schedule displayed on detail page", async () => {
- await expect(page.getByText(CRON_SCHEDULE)).toBeVisible({ timeout: 10_000 });
+ await expect(page.getByText(CRON_SCHEDULE)).toBeVisible({
+ timeout: 10_000,
+ });
});
await test.step("verify run shows COMPLETED with conversation link", async () => {
@@ -564,5 +592,4 @@ test.describe("mock-LLM automation lifecycle", () => {
}
});
});
-
});
diff --git a/tests/e2e/mock-llm/mcp/mock-llm-mcp-slack-credentials.spec.ts b/tests/e2e/mock-llm/mcp/mock-llm-mcp-slack-credentials.spec.ts
index 611fae8473..bc0b6e571a 100644
--- a/tests/e2e/mock-llm/mcp/mock-llm-mcp-slack-credentials.spec.ts
+++ b/tests/e2e/mock-llm/mcp/mock-llm-mcp-slack-credentials.spec.ts
@@ -119,7 +119,7 @@ async function openSlackInstallModal(page: Page) {
await page.goto("/mcp", { waitUntil: "domcontentloaded" });
await dismissAnalyticsModal(page);
await expect(page.getByTestId("mcp-marketplace-grid")).toBeVisible({
- timeout: 10_000,
+ timeout: 30_000,
});
await page.getByTestId("mcp-marketplace-card-slack").click();
const modal = page.getByTestId("mcp-install-modal");
diff --git a/tests/e2e/mock-llm/settings/mock-llm-profile-management.spec.ts b/tests/e2e/mock-llm/settings/mock-llm-profile-management.spec.ts
index 823221c7a7..24add6cea8 100644
--- a/tests/e2e/mock-llm/settings/mock-llm-profile-management.spec.ts
+++ b/tests/e2e/mock-llm/settings/mock-llm-profile-management.spec.ts
@@ -40,6 +40,7 @@ import {
setChatInput,
waitForPath,
createProfileViaUI,
+ editProfileViaUI,
deleteProfileIfExists,
activateProfileViaUI,
ensureMockLLMAgentProfile,
@@ -414,12 +415,22 @@ test.describe("OpenHands provider hidden base_url preservation", () => {
await dismissAnalyticsModal(page);
await waitForTestId(page, "add-llm-profile");
- await deleteProfileIfExists(page, OPENHANDS_PROFILE);
- await createProfileViaUI(page, {
- profileName: OPENHANDS_PROFILE,
- model: OPENHANDS_MODEL,
- baseUrl: CUSTOM_BASE_URL,
- });
+ const existingProfile = page
+ .getByTestId("profile-row")
+ .filter({ has: page.locator(`span[title="${OPENHANDS_PROFILE}"]`) });
+ if ((await existingProfile.count()) > 0) {
+ await editProfileViaUI(page, {
+ profileName: OPENHANDS_PROFILE,
+ model: OPENHANDS_MODEL,
+ baseUrl: CUSTOM_BASE_URL,
+ });
+ } else {
+ await createProfileViaUI(page, {
+ profileName: OPENHANDS_PROFILE,
+ model: OPENHANDS_MODEL,
+ baseUrl: CUSTOM_BASE_URL,
+ });
+ }
await test.step("open profile in edit mode", async () => {
const profileRows = page.getByTestId("profile-row");