π OpenHands: AI-Driven Development
APP-190 Add browser screenshot support for V1 conversations (#11919)
Tim O'Farrell1a983d297811
diff --git a/frontend/__tests__/conversation-websocket-handler.test.tsx b/frontend/__tests__/conversation-websocket-handler.test.tsxindex f7d67d82b5..f922a8876c 100644--- a/frontend/__tests__/conversation-websocket-handler.test.tsx+++ b/frontend/__tests__/conversation-websocket-handler.test.tsx@@ -1,12 +1,26 @@-import { describe, it, expect, beforeAll, afterAll, afterEach } from "vitest";+import {+ describe,+ it,+ expect,+ beforeAll,+ beforeEach,+ afterAll,+ afterEach,+} from "vitest";import { screen, waitFor, render, cleanup } from "@testing-library/react";import { QueryClient, QueryClientProvider } from "@tanstack/react-query";import { http, HttpResponse } from "msw";import { useOptimisticUserMessageStore } from "#/stores/optimistic-user-message-store";+import { useBrowserStore } from "#/stores/browser-store";+import { useCommandStore } from "#/state/command-store";import {createMockMessageEvent,createMockUserMessageEvent,createMockAgentErrorEvent,+ createMockBrowserObservationEvent,+ createMockBrowserNavigateActionEvent,+ createMockExecuteBashActionEvent,+ createMockExecuteBashObservationEvent,} from "#/mocks/mock-ws-helpers";import {ConnectionStatusComponent,@@ -461,7 +475,7 @@ describe("Conversation WebSocket Handler", () => {);// Create a test component that displays loading state- const HistoryLoadingComponent = () => {+ function HistoryLoadingComponent() {const context = useConversationWebSocket();const { events } = useEventStore();@@ -474,7 +488,7 @@ describe("Conversation WebSocket Handler", () => {<div data-testid="expected-event-count">{expectedEventCount}</div></div>);- };+ }// Render with WebSocket contextrenderWithWebSocketContext(@@ -484,7 +498,9 @@ describe("Conversation WebSocket Handler", () => {);// Initially should be loading history- expect(screen.getByTestId("is-loading-history")).toHaveTextContent("true");+ expect(screen.getByTestId("is-loading-history")).toHaveTextContent(+ "true",+ );// Wait for all events to be receivedawait waitFor(() => {@@ -523,7 +539,7 @@ describe("Conversation WebSocket Handler", () => {);// Create a test component that displays loading state- const HistoryLoadingComponent = () => {+ function HistoryLoadingComponent() {const context = useConversationWebSocket();return (@@ -533,7 +549,7 @@ describe("Conversation WebSocket Handler", () => {</div></div>);- };+ }// Render with WebSocket contextrenderWithWebSocketContext(@@ -583,7 +599,7 @@ describe("Conversation WebSocket Handler", () => {);// Create a test component that displays loading state- const HistoryLoadingComponent = () => {+ function HistoryLoadingComponent() {const context = useConversationWebSocket();const { events } = useEventStore();@@ -595,7 +611,7 @@ describe("Conversation WebSocket Handler", () => {<div data-testid="events-received">{events.length}</div></div>);- };+ }// Render with WebSocket contextrenderWithWebSocketContext(@@ -605,7 +621,9 @@ describe("Conversation WebSocket Handler", () => {);// Initially should be loading history- expect(screen.getByTestId("is-loading-history")).toHaveTextContent("true");+ expect(screen.getByTestId("is-loading-history")).toHaveTextContent(+ "true",+ );// Wait for all events to be receivedawait waitFor(() => {@@ -621,17 +639,133 @@ describe("Conversation WebSocket Handler", () => {});});- // 9. Terminal I/O Tests (ExecuteBashAction and ExecuteBashObservation)- describe("Terminal I/O Integration", () => {- it("should append command to store when ExecuteBashAction event is received", async () => {- const { createMockExecuteBashActionEvent } = await import(- "#/mocks/mock-ws-helpers"+ // 9. Browser State Tests (BrowserObservation)+ describe("Browser State Integration", () => {+ beforeEach(() => {+ useBrowserStore.getState().reset();+ });++ it("should update browser store with screenshot when BrowserObservation event is received", async () => {+ // Create a mock BrowserObservation event with screenshot data+ const mockBrowserObsEvent = createMockBrowserObservationEvent(+ "base64-screenshot-data",+ "Page loaded successfully",+ );++ // Set up MSW to send the event when connection is established+ mswServer.use(+ wsLink.addEventListener("connection", ({ client, server }) => {+ server.connect();+ // Send the mock event after connection+ client.send(JSON.stringify(mockBrowserObsEvent));+ }),+ );++ // Render with WebSocket context+ renderWithWebSocketContext(<ConnectionStatusComponent />);++ // Wait for connection+ await waitFor(() => {+ expect(screen.getByTestId("connection-state")).toHaveTextContent(+ "OPEN",+ );+ });++ // Wait for the browser store to be updated with screenshot+ await waitFor(() => {+ const { screenshotSrc } = useBrowserStore.getState();+ expect(screenshotSrc).toBe(+ "data:image/png;base64,base64-screenshot-data",+ );+ });+ });++ it("should update browser store with URL when BrowserNavigateAction followed by BrowserObservation", async () => {+ // Create mock events - action first, then observation+ const mockBrowserActionEvent = createMockBrowserNavigateActionEvent(+ "https://example.com/test-page",+ );+ const mockBrowserObsEvent = createMockBrowserObservationEvent(+ "base64-screenshot-data",+ "Page loaded successfully",);- const { useCommandStore } = await import("#/state/command-store");- // Clear the command store before test+ // Set up MSW to send both events when connection is established+ mswServer.use(+ wsLink.addEventListener("connection", ({ client, server }) => {+ server.connect();+ // Send action first, then observation+ client.send(JSON.stringify(mockBrowserActionEvent));+ client.send(JSON.stringify(mockBrowserObsEvent));+ }),+ );++ // Render with WebSocket context+ renderWithWebSocketContext(<ConnectionStatusComponent />);++ // Wait for connection+ await waitFor(() => {+ expect(screen.getByTestId("connection-state")).toHaveTextContent(+ "OPEN",+ );+ });++ // Wait for the browser store to be updated with both screenshot and URL+ await waitFor(() => {+ const { screenshotSrc, url } = useBrowserStore.getState();+ expect(screenshotSrc).toBe(+ "data:image/png;base64,base64-screenshot-data",+ );+ expect(url).toBe("https://example.com/test-page");+ });+ });++ it("should not update browser store when BrowserObservation has no screenshot data", async () => {+ const initialScreenshot = useBrowserStore.getState().screenshotSrc;++ // Create a mock BrowserObservation event WITHOUT screenshot data+ const mockBrowserObsEvent = createMockBrowserObservationEvent(+ null, // no screenshot+ "Browser action completed",+ );++ // Set up MSW to send the event when connection is established+ mswServer.use(+ wsLink.addEventListener("connection", ({ client, server }) => {+ server.connect();+ // Send the mock event after connection+ client.send(JSON.stringify(mockBrowserObsEvent));+ }),+ );++ // Render with WebSocket context+ renderWithWebSocketContext(<ConnectionStatusComponent />);++ // Wait for connection+ await waitFor(() => {+ expect(screen.getByTestId("connection-state")).toHaveTextContent(+ "OPEN",+ );+ });++ // Give some time for any potential updates+ await new Promise((resolve) => {+ setTimeout(resolve, 100);+ });++ // Screenshot should remain unchanged (empty/initial value)+ const { screenshotSrc } = useBrowserStore.getState();+ expect(screenshotSrc).toBe(initialScreenshot);+ });+ });++ // 10. Terminal I/O Tests (ExecuteBashAction and ExecuteBashObservation)+ describe("Terminal I/O Integration", () => {+ beforeEach(() => {useCommandStore.getState().clearTerminal();+ });+ it("should append command to store when ExecuteBashAction event is received", async () => {// Create a mock ExecuteBashAction eventconst mockBashActionEvent = createMockExecuteBashActionEvent("npm test");@@ -667,14 +801,6 @@ describe("Conversation WebSocket Handler", () => {});it("should append output to store when ExecuteBashObservation event is received", async () => {- const { createMockExecuteBashObservationEvent } = await import(- "#/mocks/mock-ws-helpers"- );- const { useCommandStore } = await import("#/state/command-store");-- // Clear the command store before test- useCommandStore.getState().clearTerminal();-// Create a mock ExecuteBashObservation eventconst mockBashObservationEvent = createMockExecuteBashObservationEvent("PASS tests/example.test.js\n β should work (2 ms)",diff --git a/frontend/src/components/v1/chat/event-content-helpers/__tests__/get-observation-content.test.ts b/frontend/src/components/v1/chat/event-content-helpers/__tests__/get-observation-content.test.tsnew file mode 100644index 0000000000..d35dc97925--- /dev/null+++ b/frontend/src/components/v1/chat/event-content-helpers/__tests__/get-observation-content.test.ts@@ -0,0 +1,92 @@+import { describe, it, expect } from "vitest";+import { getObservationContent } from "../get-observation-content";+import { ObservationEvent } from "#/types/v1/core";+import { BrowserObservation } from "#/types/v1/core/base/observation";++describe("getObservationContent - BrowserObservation", () => {+ it("should return output content when available", () => {+ const mockEvent: ObservationEvent<BrowserObservation> = {+ id: "test-id",+ timestamp: "2024-01-01T00:00:00Z",+ source: "environment",+ tool_name: "browser_navigate",+ tool_call_id: "call-id",+ action_id: "action-id",+ observation: {+ kind: "BrowserObservation",+ output: "Browser action completed",+ error: null,+ screenshot_data: "base64data",+ },+ };++ const result = getObservationContent(mockEvent);++ expect(result).toContain("**Output:**");+ expect(result).toContain("Browser action completed");+ });++ it("should handle error cases properly", () => {+ const mockEvent: ObservationEvent<BrowserObservation> = {+ id: "test-id",+ timestamp: "2024-01-01T00:00:00Z",+ source: "environment",+ tool_name: "browser_navigate",+ tool_call_id: "call-id",+ action_id: "action-id",+ observation: {+ kind: "BrowserObservation",+ output: "",+ error: "Browser action failed",+ screenshot_data: null,+ },+ };++ const result = getObservationContent(mockEvent);++ expect(result).toContain("**Error:**");+ expect(result).toContain("Browser action failed");+ });++ it("should provide default message when no output or error", () => {+ const mockEvent: ObservationEvent<BrowserObservation> = {+ id: "test-id",+ timestamp: "2024-01-01T00:00:00Z",+ source: "environment",+ tool_name: "browser_navigate",+ tool_call_id: "call-id",+ action_id: "action-id",+ observation: {+ kind: "BrowserObservation",+ output: "",+ error: null,+ screenshot_data: "base64data",+ },+ };++ const result = getObservationContent(mockEvent);++ expect(result).toBe("Browser action completed successfully.");+ });++ it("should return output when screenshot_data is null", () => {+ const mockEvent: ObservationEvent<BrowserObservation> = {+ id: "test-id",+ timestamp: "2024-01-01T00:00:00Z",+ source: "environment",+ tool_name: "browser_navigate",+ tool_call_id: "call-id",+ action_id: "action-id",+ observation: {+ kind: "BrowserObservation",+ output: "Page loaded successfully",+ error: null,+ screenshot_data: null,+ },+ };++ const result = getObservationContent(mockEvent);++ expect(result).toBe("**Output:**\nPage loaded successfully");+ });+});diff --git a/frontend/src/components/v1/chat/event-content-helpers/get-observation-content.ts b/frontend/src/components/v1/chat/event-content-helpers/get-observation-content.tsindex da2558af42..bf443ea71c 100644--- a/frontend/src/components/v1/chat/event-content-helpers/get-observation-content.ts+++ b/frontend/src/components/v1/chat/event-content-helpers/get-observation-content.ts@@ -98,14 +98,16 @@ const getBrowserObservationContent = (.filter((c) => c.type === "text").map((c) => c.text).join("\n")- : "";+ : observation.output || "";let contentDetails = "";- if ("is_error" in observation && observation.is_error) {- contentDetails += `**Error:**\n${textContent}`;- } else {+ if (observation.error) {+ contentDetails += `**Error:**\n${observation.error}`;+ } else if (textContent) {contentDetails += `**Output:**\n${textContent}`;+ } else {+ contentDetails += "Browser action completed successfully.";}if (contentDetails.length > MAX_CONTENT_LENGTH) {diff --git a/frontend/src/components/v1/chat/event-message.tsx b/frontend/src/components/v1/chat/event-message.tsxindex ce0380c9c8..95690d8984 100644--- a/frontend/src/components/v1/chat/event-message.tsx+++ b/frontend/src/components/v1/chat/event-message.tsx@@ -118,7 +118,7 @@ const renderUserMessageWithSkillReady = ();} catch (error) {// If skill ready event creation fails, just render the user message- console.error("Failed to create skill ready event:", error);+ // Failed to create skill ready event, fallback to user messagereturn (<UserAssistantEventMessageevent={messageEvent}diff --git a/frontend/src/contexts/conversation-websocket-context.tsx b/frontend/src/contexts/conversation-websocket-context.tsxindex ae115c41a9..68c50f9499 100644--- a/frontend/src/contexts/conversation-websocket-context.tsx+++ b/frontend/src/contexts/conversation-websocket-context.tsx@@ -14,6 +14,7 @@ import { useErrorMessageStore } from "#/stores/error-message-store";import { useOptimisticUserMessageStore } from "#/stores/optimistic-user-message-store";import { useV1ConversationStateStore } from "#/stores/v1-conversation-state-store";import { useCommandStore } from "#/state/command-store";+import { useBrowserStore } from "#/stores/browser-store";import {isV1Event,isAgentErrorEvent,@@ -27,6 +28,8 @@ import {isExecuteBashObservationEvent,isConversationErrorEvent,isPlanningFileEditorObservationEvent,+ isBrowserObservationEvent,+ isBrowserNavigateActionEvent,} from "#/types/v1/type-guards";import { ConversationStateUpdateEventStats } from "#/types/v1/core/events/conversation-state-event";import { handleActionEventCacheInvalidation } from "#/utils/cache-utils";@@ -383,6 +386,22 @@ export function ConversationWebSocketProvider({.join("\n");appendOutput(textContent);}++ // Handle BrowserObservation events - update browser store with screenshot+ if (isBrowserObservationEvent(event)) {+ const { screenshot_data: screenshotData } = event.observation;+ if (screenshotData) {+ const screenshotSrc = screenshotData.startsWith("data:")+ ? screenshotData+ : `data:image/png;base64,${screenshotData}`;+ useBrowserStore.getState().setScreenshotSrc(screenshotSrc);+ }+ }++ // Handle BrowserNavigateAction events - update browser store with URL+ if (isBrowserNavigateActionEvent(event)) {+ useBrowserStore.getState().setUrl(event.action.url);+ }}} catch (error) {// eslint-disable-next-line no-consolediff --git a/frontend/src/mocks/mock-ws-helpers.ts b/frontend/src/mocks/mock-ws-helpers.tsindex 512045766f..ae4e214943 100644--- a/frontend/src/mocks/mock-ws-helpers.ts+++ b/frontend/src/mocks/mock-ws-helpers.ts@@ -184,3 +184,55 @@ export const createMockExecuteBashObservationEvent = (},action_id: "bash-action-123",});++/**+ * Creates a mock BrowserObservation event for testing browser state handling+ */+export const createMockBrowserObservationEvent = (+ screenshotData: string | null = "base64-screenshot-data",+ output: string = "Browser action completed",+ error: string | null = null,+) => ({+ id: "browser-obs-123",+ timestamp: new Date().toISOString(),+ source: "environment",+ tool_name: "browser_navigate",+ tool_call_id: "browser-call-456",+ observation: {+ kind: "BrowserObservation",+ output,+ error,+ screenshot_data: screenshotData,+ },+ action_id: "browser-action-123",+});++/**+ * Creates a mock BrowserNavigateAction event for testing browser URL extraction+ */+export const createMockBrowserNavigateActionEvent = (+ url: string = "https://example.com",+) => ({+ id: "browser-action-123",+ timestamp: new Date().toISOString(),+ source: "agent",+ thought: [{ type: "text", text: "Navigating to URL" }],+ thinking_blocks: [],+ action: {+ kind: "BrowserNavigateAction",+ url,+ new_tab: false,+ },+ tool_name: "browser_navigate",+ tool_call_id: "browser-call-456",+ tool_call: {+ id: "browser-call-456",+ type: "function",+ function: {+ name: "browser_navigate",+ arguments: JSON.stringify({ url, new_tab: false }),+ },+ },+ llm_response_id: "llm-response-789",+ security_risk: { level: "low" },+});diff --git a/frontend/src/types/v1/type-guards.ts b/frontend/src/types/v1/type-guards.tsindex 306661e854..ee831ea489 100644--- a/frontend/src/types/v1/type-guards.ts+++ b/frontend/src/types/v1/type-guards.ts@@ -7,6 +7,8 @@ import {ExecuteBashObservation,PlanningFileEditorObservation,TerminalObservation,+ BrowserObservation,+ BrowserNavigateAction,} from "./core";import { AgentErrorEvent } from "./core/events/observation-event";import { MessageEvent } from "./core/events/message-event";@@ -126,6 +128,22 @@ export const isPlanningFileEditorObservationEvent = (isObservationEvent(event) &&event.observation.kind === "PlanningFileEditorObservation";+/**+ * Type guard function to check if an observation event is a BrowserObservation+ */+export const isBrowserObservationEvent = (+ event: OpenHandsEvent,+): event is ObservationEvent<BrowserObservation> =>+ isObservationEvent(event) && event.observation.kind === "BrowserObservation";++/**+ * Type guard function to check if an action event is a BrowserNavigateAction+ */+export const isBrowserNavigateActionEvent = (+ event: OpenHandsEvent,+): event is ActionEvent<BrowserNavigateAction> =>+ isActionEvent(event) && event.action.kind === "BrowserNavigateAction";+/*** Type guard function to check if an event is a system prompt event*/