Skip to content

Self-improving AI Employees for everyone

feat(workspace): show native browser in normal and dev modes

trohitg681a340fb19c
diff --git a/client/src/components/browser/BrowserWorkspace.tsx b/client/src/components/browser/BrowserWorkspace.tsx
new file mode 100644
index 00000000..fd1f4f28
--- /dev/null
+++ b/client/src/components/browser/BrowserWorkspace.tsx
@@ -0,0 +1,357 @@
+import { useCallback, useEffect, useRef, useState } from 'react';
+import type { KeyboardEvent, PointerEvent } from 'react';
+import { ArrowLeft, ArrowRight, Globe, RotateCw } from 'lucide-react';
+import { Button } from '@/components/ui/button';
+import { buildApiUrl } from '@/config/api';
+import { useWebSocketActions } from '@/contexts/WebSocketContext';
+import { browserModifiers, browserPoint, decodeBrowserFrame, type BrowserFrameHeader } from './protocol';
+
+export interface BrowserWorkspaceProps {
+ workflowId?: string | null;
+ nodes: { node_id: string; label: string }[];
+ visible?: boolean;
+}
+type Phase = 'connecting' | 'idle' | 'live' | 'error';
+interface BrowserState {
+ state: string;
+ controller?: string | null;
+ request?: { message?: string; reason?: string } | null;
+}
+interface Tab { target_id: string; title?: string; url?: string; active?: boolean }
+const inputClass = 'min-w-0 rounded border border-border-default bg-bg-panel px-2 py-1 text-xs text-fg-default outline-none focus-visible:ring-2 focus-visible:ring-ring';
+
+export default function BrowserWorkspace({ workflowId, nodes, visible = true }: BrowserWorkspaceProps) {
+ const [selected, setSelected] = useState('');
+ const nodeId = nodes.some((n) => n.node_id === selected) ? selected : nodes[0]?.node_id;
+ if (!workflowId) return <EmptyBrowser message="Save this workflow to open its browser." />;
+ if (!nodeId) return <EmptyBrowser message="Add a Browser node to this workflow to watch it here." />;
+ return (
+ <div className="flex min-h-0 min-w-0 flex-1 flex-col gap-2">
+ {nodes.length > 1 && (
+ <select aria-label="Browser node" className={inputClass} value={nodeId} onChange={(e) => setSelected(e.target.value)}>
+ {nodes.map((node) => <option key={node.node_id} value={node.node_id}>{node.label || 'Browser'}</option>)}
+ </select>
+ )}
+ <BrowserSessionView key={`${workflowId}:${nodeId}`} workflowId={workflowId} nodeId={nodeId} visible={visible} />
+ </div>
+ );
+}
+
+function EmptyBrowser({ message }: { message: string }) {
+ return <div className="m-auto flex max-w-80 flex-col items-center gap-3 p-6 text-center text-sm text-fg-muted"><Globe aria-hidden className="size-6" />{message}</div>;
+}
+
+function BrowserSessionView({ workflowId, nodeId, visible }: { workflowId: string; nodeId: string; visible: boolean }) {
+ const { sendRequest, isReady } = useWebSocketActions();
+ const socketRef = useRef<WebSocket | null>(null);
+ const surfaceRef = useRef<HTMLDivElement>(null);
+ const canvasRef = useRef<HTMLCanvasElement>(null);
+ const visibleRef = useRef(visible);
+ visibleRef.current = visible;
+ const frameRef = useRef<{ header: BrowserFrameHeader; width: number; height: number } | null>(null);
+ const [phase, setPhase] = useState<Phase>('connecting');
+ const [state, setState] = useState<BrowserState>({ state: 'idle' });
+ const [error, setErrorText] = useState('');
+ const streamErrorRef = useRef(false);
+ const setError = useCallback((message: string, streamError = false) => {
+ streamErrorRef.current = streamError;
+ setErrorText(message);
+ }, []);
+ const [tabs, setTabs] = useState<Tab[]>([]);
+ const [address, setAddress] = useState('');
+ const [hasFrame, setHasFrame] = useState(false);
+ const [starting, setStarting] = useState(false);
+ const [installing, setInstalling] = useState(false);
+ const [attempt, setAttempt] = useState(0);
+ const [dialog, setDialog] = useState<{ kind: string; message: string } | null>(null);
+ const [promptText, setPromptText] = useState('');
+ const [takingControl, setTakingControl] = useState(false);
+ const control = phase === 'live' && state.state === 'user' && state.controller === 'you';
+ const controlRef = useRef(control);
+ controlRef.current = control;
+ const mountedRef = useRef(true);
+ useEffect(() => { mountedRef.current = true; return () => { mountedRef.current = false; }; }, []);
+ const send = useCallback((message: Record<string, unknown>) => {
+ const socket = socketRef.current;
+ if (socket?.readyState === WebSocket.OPEN) socket.send(JSON.stringify(message));
+ }, []);
+
+ const releaseControl = useCallback(() => {
+ controlRef.current = false;
+ setTakingControl(false);
+ setState((previous) => ({ ...previous, controller: null }));
+ send({ type: 'control_release' });
+ }, [send]);
+
+ useEffect(() => {
+ if (!isReady) { setPhase('connecting'); return; }
+ let disposed = false;
+ let retryTimer: ReturnType<typeof setTimeout> | undefined;
+ let pendingImage: HTMLImageElement | null = null;
+ const urls = new Set<string>();
+ let chain = Promise.resolve();
+ let queuedFrames = 0;
+ // Opt-in numeric summaries only: no page, frame, session or input payloads.
+ const debug = new URLSearchParams(window.location.search).get('browserStreamDebug') === '1';
+ const startedAt = performance.now();
+ let paintedFrames = 0, decodeTotal = 0, drawTotal = 0;
+ let terminalStreamError = false;
+ let streamEpoch = 0;
+ const viewport = () => {
+ const rect = surfaceRef.current?.getBoundingClientRect();
+ return { width: Math.max(80, rect?.width || 800), height: Math.max(60, rect?.height || 500), dpr: window.devicePixelRatio || 1 };
+ };
+ const effectiveVisible = () => visibleRef.current && document.visibilityState !== 'hidden';
+ const transmit = (message: Record<string, unknown>) => {
+ if (!disposed && socket.readyState === WebSocket.OPEN) socket.send(JSON.stringify(message));
+ };
+ setPhase('connecting'); setError(''); setHasFrame(false); frameRef.current = null;
+ setState({ state: 'idle' }); setTabs([]); setAddress(''); setDialog(null); setTakingControl(false);
+ const url = new URL(buildApiUrl('/ws/browser'), window.location.href);
+ url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
+ const socket = new WebSocket(url.toString());
+ socket.binaryType = 'arraybuffer';
+ socketRef.current = socket;
+ socket.onopen = () => transmit({ type: 'attach', target: { kind: 'node', workflow_id: workflowId, node_id: nodeId }, viewport: viewport(), visible: effectiveVisible(), max_fps: 30 });
+ socket.onmessage = (event) => {
+ if (disposed) return;
+ if (typeof event.data === 'string') {
+ try {
+ const message = JSON.parse(event.data);
+ if (message.type === 'attached') setPhase(message.running ? 'live' : 'idle');
+ if (message.type === 'state') { setState(message); if (!terminalStreamError) setPhase('live'); setTakingControl(false); }
+ if (message.type === 'idle') { streamEpoch += 1; setPhase('idle'); setState({ state: 'idle' }); setHasFrame(false); frameRef.current = null; setDialog(null); }
+ if (message.type === 'page') setAddress(message.url || '');
+ if (message.type === 'tabs') setTabs(message.tabs || []);
+ if (message.type === 'dialog') { setDialog(message); setPromptText(''); }
+ if (message.type === 'error') {
+ setError(message.message || 'Browser request failed.', message.code === 'screencast');
+ if (message.code === 'screencast' && message.retrying === false) {
+ terminalStreamError = true; streamEpoch += 1;
+ setPhase('error'); setHasFrame(false); frameRef.current = null;
+ }
+ }
+ if (message.type === 'control') {
+ setTakingControl(false);
+ if (!message.granted) setError(message.message || (message.reason === 'taken_over_elsewhere' ? 'Control moved to another viewer.' : 'Control could not be granted. Try again.'));
+ }
+ } catch { setError('The browser sent an invalid status message.'); }
+ return;
+ }
+ // Serial decoding keeps older frames from painting over newer ones. Every
+ // frame is acknowledged, including hidden frames and decode failures.
+ const data = event.data as ArrayBuffer;
+ const frameEpoch = streamEpoch;
+ // The server normally permits two outstanding frames. Keep that bound
+ // locally too if a faulty sender exceeds its credit; ACK dropped frames.
+ if (queuedFrames >= 2) {
+ let seq: number | undefined;
+ try { seq = decodeBrowserFrame(data).header.seq; } catch { /* ACK malformed input too. */ }
+ transmit({ type: 'ack', seq });
+ return;
+ }
+ queuedFrames += 1;
+ chain = chain.then(async () => {
+ let seq: number | undefined;
+ let objectUrl: string | undefined;
+ try {
+ if (disposed) return;
+ const frame = decodeBrowserFrame(data); seq = frame.header.seq;
+ if (frameEpoch !== streamEpoch) return;
+ if (!effectiveVisible()) return;
+ objectUrl = URL.createObjectURL(frame.jpeg); urls.add(objectUrl);
+ const decodeStarted = performance.now();
+ const image = new Image(); pendingImage = image;
+ await new Promise<void>((resolve, reject) => {
+ image.onload = () => resolve(); image.onerror = () => reject(new Error('Could not decode browser frame.'));
+ image.src = objectUrl!;
+ });
+ if (disposed || frameEpoch !== streamEpoch || !effectiveVisible()) return;
+ const decodedAt = performance.now();
+ const canvas = canvasRef.current;
+ if (!canvas) return;
+ const context = canvas.getContext('2d');
+ if (!context) throw new Error('This browser cannot display the live view.');
+ if (canvas.width !== image.naturalWidth) canvas.width = image.naturalWidth;
+ if (canvas.height !== image.naturalHeight) canvas.height = image.naturalHeight;
+ context.drawImage(image, 0, 0);
+ if (debug) {
+ paintedFrames += 1;
+ decodeTotal += decodedAt - decodeStarted;
+ drawTotal += performance.now() - decodedAt;
+ if (paintedFrames === 1 || paintedFrames % 60 === 0) {
+ console.debug('[browser-stream]', {
+ frames: paintedFrames,
+ ...(paintedFrames === 1 ? { firstFrameMs: performance.now() - startedAt } : {}),
+ meanDecodeMs: decodeTotal / paintedFrames, meanDrawMs: drawTotal / paintedFrames,
+ });
+ }
+ }
+ frameRef.current = { header: frame.header, width: image.naturalWidth, height: image.naturalHeight };
+ setHasFrame(true); setPhase('live');
+ terminalStreamError = false;
+ if (streamErrorRef.current) setError('');
+ } catch (cause) {
+ if (!disposed) setError(cause instanceof Error ? cause.message : 'Could not read browser frame.', true);
+ } finally {
+ queuedFrames -= 1;
+ pendingImage = null;
+ if (objectUrl) { URL.revokeObjectURL(objectUrl); urls.delete(objectUrl); }
+ transmit({ type: 'ack', seq });
+ }
+ });
+ };
+ socket.onerror = () => { if (!disposed) setError('Browser connection interrupted.'); };
+ socket.onclose = (event) => {
+ if (disposed) return;
+ streamEpoch += 1;
+ setPhase('error'); setHasFrame(false); frameRef.current = null; setTakingControl(false);
+ setError(event.reason || 'Browser connection lost.');
+ // Authentication/ownership failures need user action, not a retry loop.
+ if (![4001, 4002, 4003, 4004].includes(event.code)) retryTimer = setTimeout(() => setAttempt((n) => n + 1), 2000);
+ };
+ const observer = new ResizeObserver(() => transmit({ type: 'viewport', ...viewport() }));
+ if (surfaceRef.current) observer.observe(surfaceRef.current);
+ const onVisibility = () => {
+ transmit({ type: 'visibility', visible: effectiveVisible() });
+ if (!effectiveVisible()) releaseControl();
+ };
+ window.addEventListener('blur', releaseControl);
+ document.addEventListener('visibilitychange', onVisibility);
+ return () => {
+ disposed = true; window.removeEventListener('blur', releaseControl); clearTimeout(retryTimer); observer.disconnect(); document.removeEventListener('visibilitychange', onVisibility);
+ if (socket.readyState === WebSocket.OPEN) socket.send(JSON.stringify({ type: 'control_release' }));
+ socket.close(); if (socketRef.current === socket) socketRef.current = null;
+ if (pendingImage) { pendingImage.onerror?.(new Event('error')); pendingImage.src = ''; }
+ for (const objectUrl of urls) URL.revokeObjectURL(objectUrl);
+ urls.clear();
+ };
+ }, [workflowId, nodeId, isReady, attempt, setError, releaseControl]);
+
+ useEffect(() => {
+ send({ type: 'visibility', visible: visible && document.visibilityState !== 'hidden' });
+ if (!visible) releaseControl();
+ }, [visible, send, releaseControl]);
+
+ // Installer work outlives the finite open request; report that state and let
+ // the owner retry once ready. Do not launch a browser merely by viewing it.
+ useEffect(() => {
+ if (!isReady || !visible || (!starting && !installing && phase !== 'idle')) return;
+ let disposed = false;
+ let inFlight = false;
+ const check = async () => {
+ if (inFlight) return;
+ inFlight = true;
+ try {
+ const status = await sendRequest<{ chrome?: { phase?: string }; cli?: { phase?: string } }>('browser_runtime_status');
+ if (!disposed) setInstalling([status.chrome?.phase, status.cli?.phase].some((s) => s === 'installing' || s === 'downloading' || s === 'extracting'));
+ } catch { /* The open request supplies actionable errors. */ }
+ finally { inFlight = false; }
+ };
+ void check(); const timer = setInterval(() => void check(), 3000);
+ return () => { disposed = true; clearInterval(timer); };
+ }, [starting, installing, phase, visible, isReady, sendRequest]);
+
+ const start = async () => {
+ setStarting(true); setError('');
+ try {
+ const result = await sendRequest<{ success: boolean; error?: string }>('browser_session_open', { workflow_id: workflowId, node_id: nodeId }, 60_000);
+ if (!result.success) throw new Error(result.error || 'Could not start the browser.');
+ if (mountedRef.current) setAttempt((n) => n + 1);
+ } catch (cause) {
+ if (mountedRef.current) setError(cause instanceof Error ? cause.message : 'Could not start the browser.');
+ } finally { if (mountedRef.current) setStarting(false); }
+ };
+ const point = (clientX: number, clientY: number) => {
+ const rect = surfaceRef.current?.getBoundingClientRect(); const frame = frameRef.current;
+ return rect && frame ? browserPoint(clientX - rect.left, clientY - rect.top, rect.width, rect.height, frame.width, frame.height, frame.header) : null;
+ };
+ useEffect(() => {
+ const surface = surfaceRef.current;
+ if (!surface) return;
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ let pending: { x: number; y: number; delta_x: number; delta_y: number; modifiers: number } | null = null;
+ const wheel = (event: WheelEvent) => {
+ if (!controlRef.current) return;
+ const frame = frameRef.current; if (!frame) return;
+ const rect = surface.getBoundingClientRect();
+ const coords = browserPoint(event.clientX - rect.left, event.clientY - rect.top, rect.width, rect.height, frame.width, frame.height, frame.header);
+ if (!coords) return;
+ event.preventDefault(); event.stopPropagation();
+ const factor = event.deltaMode === 1 ? 16 : event.deltaMode === 2 ? rect.height : 1;
+ pending = { ...coords, delta_x: (pending?.delta_x || 0) + event.deltaX * factor, delta_y: (pending?.delta_y || 0) + event.deltaY * factor, modifiers: browserModifiers(event) };
+ if (timer) return;
+ timer = setTimeout(() => { if (pending && controlRef.current) send({ type: 'wheel', ...pending }); pending = null; timer = undefined; }, 25);
+ };
+ // React wheel listeners are passive; a native listener prevents scrolling
+ // the surrounding app while the user scrolls the remote page.
+ surface.addEventListener('wheel', wheel, { passive: false });
+ return () => { surface.removeEventListener('wheel', wheel); clearTimeout(timer); };
+ }, [send]);
+ const lastMove = useRef(0);
+ const lastPointer = useRef<{ x: number; y: number } | null>(null);
+ const pointer = (event: PointerEvent<HTMLDivElement>, action: 'move' | 'down' | 'up') => {
+ if (!control) return;
+ const coords = point(event.clientX, event.clientY) || (action === 'up' ? lastPointer.current : null);
+ if (!coords) return;
+ lastPointer.current = coords;
+ if (action === 'move' && Date.now() - lastMove.current < 25) return;
+ lastMove.current = Date.now(); event.preventDefault();
+ if (action === 'down') { event.currentTarget.focus(); event.currentTarget.setPointerCapture?.(event.pointerId); }
+ if (action === 'up' && event.currentTarget.hasPointerCapture?.(event.pointerId)) event.currentTarget.releasePointerCapture(event.pointerId);
+ send({ type: 'mouse', action, ...coords, button: ['left', 'middle', 'right'][event.button] || 'left', buttons: event.buttons, click_count: Math.max(1, event.detail), modifiers: browserModifiers(event) });
+ };
+ const key = (event: KeyboardEvent<HTMLDivElement>, action: 'down' | 'up') => {
+ if (!control || event.nativeEvent.isComposing) return;
+ // Let the browser's paste event expose clipboard text; do not also send V.
+ if ((event.ctrlKey || event.metaKey) && event.key.toLowerCase() === 'v') return;
+ event.preventDefault(); event.stopPropagation();
+ send({ type: 'key', action, key: event.key, code: event.code, key_code: event.keyCode, location: event.location, repeat: event.repeat, modifiers: browserModifiers(event) });
+ };
+ const navigate = (action: string) => send({ type: 'navigate', action, ...(action === 'goto' ? { url: address } : {}) });
+ const statusLabel = phase === 'connecting' ? 'Connecting…' : phase === 'idle' ? 'Browser is stopped' : phase === 'error' ? 'Disconnected' : control ? 'You have control' : state.state === 'agent' ? 'Employee is browsing' : state.state === 'awaiting_user' ? 'Your help is needed' : state.controller === 'other' ? 'Another viewer has control' : 'Live browser';
+
+ return (
+ <div className="flex min-h-0 min-w-0 flex-1 flex-col gap-2">
+ <div className="flex flex-wrap items-center justify-between gap-2 text-xs">
+ <span role="status" className="text-fg-muted">{installing ? 'Installing browser…' : starting ? 'Starting browser…' : statusLabel}</span>
+ {phase === 'live' && <Button size="sm" variant="outline" disabled={takingControl} onClick={() => {
+ setError('');
+ if (control) send({ type: 'control_release' });
+ else { setTakingControl(true); send({ type: 'control_request', ...(state.controller === 'other' ? { force: true } : {}) }); }
+ }}>{control ? 'Hand back' : takingControl ? 'Taking control…' : state.controller === 'other' ? 'Take over here' : 'Take control'}</Button>}
+ </div>
+ {state.request && <div className="rounded border border-border-default p-2 text-sm text-fg-default">{state.request.message || state.request.reason || 'The employee needs your help.'}</div>}
+ {error && <div role="alert" className="flex items-start gap-2 text-xs text-destructive"><span className="min-w-0 flex-1 break-words">{error}</span><button aria-label="Dismiss browser error" onClick={() => setError('')}>×</button></div>}
+ <form className="flex items-center gap-1" onSubmit={(e) => { e.preventDefault(); if (control) navigate('goto'); }}>
+ <Button type="button" size="icon-sm" variant="ghost" aria-label="Back" disabled={!control} onClick={() => navigate('back')}><ArrowLeft className="size-4" /></Button>
+ <Button type="button" size="icon-sm" variant="ghost" aria-label="Forward" disabled={!control} onClick={() => navigate('forward')}><ArrowRight className="size-4" /></Button>
+ <Button type="button" size="icon-sm" variant="ghost" aria-label="Reload page" disabled={!control} onClick={() => navigate('reload')}><RotateCw className="size-4" /></Button>
+ <input aria-label="Browser address" className={`${inputClass} flex-1`} value={address} disabled={!control} placeholder="Enter a website" onChange={(e) => setAddress(e.target.value)} />
+ </form>
+ {tabs.length > 1 && <select aria-label="Browser tab" className={inputClass} disabled={!control} value={tabs.find((tab) => tab.active)?.target_id || ''} onChange={(e) => send({ type: 'tab', action: 'activate', target_id: e.target.value })}>
+ {tabs.map((tab) => <option key={tab.target_id} value={tab.target_id}>{tab.title || tab.url || 'New tab'}</option>)}
+ </select>}
+ <div ref={surfaceRef} role="group" aria-label="Live browser page" tabIndex={control ? 0 : -1}
+ className="relative flex min-h-40 flex-1 items-center justify-center overflow-hidden rounded border border-border-default bg-bg-app outline-none focus-visible:ring-2 focus-visible:ring-ring"
+ style={{ touchAction: control ? 'none' : 'auto' }}
+ onPointerDown={(e) => pointer(e, 'down')} onPointerMove={(e) => pointer(e, 'move')} onPointerUp={(e) => pointer(e, 'up')} onPointerCancel={releaseControl}
+ onContextMenu={(e) => { if (control) e.preventDefault(); }} onKeyDown={(e) => key(e, 'down')} onKeyUp={(e) => key(e, 'up')}
+ onPaste={(e) => { if (control) { e.preventDefault(); send({ type: 'insert_text', text: e.clipboardData.getData('text/plain') }); } }}
+ onCompositionEnd={(e) => { if (control && e.data) send({ type: 'insert_text', text: e.data }); }}>
+ <canvas ref={canvasRef} aria-label="Browser screenshot" className="absolute inset-0 h-full w-full object-contain" style={{ visibility: hasFrame ? 'visible' : 'hidden' }} />
+ {!hasFrame && <div className="relative flex max-w-80 flex-col items-center gap-3 p-6 text-center text-sm text-fg-muted"><Globe aria-hidden className="size-6" />
+ <span>{phase === 'live' ? 'Waiting for the browser picture…' : phase === 'idle' ? 'The browser will appear when the employee uses it. You can also open it now.' : statusLabel}</span>
+ {phase === 'idle' && <Button variant="outline" disabled={starting || installing || !isReady} onClick={() => void start()}>{starting ? 'Starting…' : installing ? 'Installing…' : 'Start browser'}</Button>}
+ {phase === 'error' && <Button variant="outline" onClick={() => setAttempt((n) => n + 1)}>Reconnect</Button>}
+ </div>}
+ </div>
+ {dialog && <div role="dialog" aria-label="Browser page dialog" className="flex flex-col gap-2 rounded border border-border-default bg-bg-panel p-3 text-sm">
+ <p className="break-words text-fg-default">{dialog.message}</p>
+ {dialog.kind === 'prompt' && <input aria-label="Dialog response" className={inputClass} value={promptText} onChange={(e) => setPromptText(e.target.value)} disabled={!control} />}
+ <div className="flex gap-2">{[true, false].map((accept) => <Button key={String(accept)} size="sm" variant="outline" disabled={!control} onClick={() => { send({ type: 'dialog_reply', accept, prompt_text: promptText }); setDialog(null); }}>{accept ? 'OK' : 'Cancel'}</Button>)}</div>
+ {!control && <span className="text-xs text-fg-muted">Take control to respond.</span>}
+ </div>}
+ </div>
+ );
+}
diff --git a/client/src/components/browser/__tests__/BrowserWorkspace.test.tsx b/client/src/components/browser/__tests__/BrowserWorkspace.test.tsx
new file mode 100644
index 00000000..e2ca0123
--- /dev/null
+++ b/client/src/components/browser/__tests__/BrowserWorkspace.test.tsx
@@ -0,0 +1,238 @@
+import { act, fireEvent, render, screen, waitFor } from '@testing-library/react';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+const { actions } = vi.hoisted(() => ({ actions: { isReady: true, sendRequest: vi.fn() } }));
+vi.mock('@/contexts/WebSocketContext', () => ({ useWebSocketActions: () => actions }));
+import BrowserWorkspace from '../BrowserWorkspace';
+
+class MockSocket {
+ static OPEN = 1;
+ static instances: MockSocket[] = [];
+ readyState = 0;
+ binaryType = '';
+ onopen: (() => void) | null = null;
+ onmessage: ((event: { data: unknown }) => void) | null = null;
+ onclose: ((event: { code: number; reason: string }) => void) | null = null;
+ onerror: (() => void) | null = null;
+ sent: Record<string, unknown>[] = [];
+ close = vi.fn(() => { this.readyState = 3; });
+ constructor(public url: string) { MockSocket.instances.push(this); }
+ send(message: string) { this.sent.push(JSON.parse(message)); }
+ open() { this.readyState = 1; this.onopen?.(); }
+ message(data: Record<string, unknown>) { this.onmessage?.({ data: JSON.stringify(data) }); }
+}
+const nodes = [{ node_id: 'browser-1', label: 'Research' }];
+
+function mockFrameRenderer(deferred = false) {
+ const loads: (() => void)[] = [];
+ const draw = vi.fn();
+ vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue({ drawImage: draw } as unknown as CanvasRenderingContext2D);
+ const revoke = vi.fn(); const create = vi.fn(() => 'blob:browser-frame');
+ const NativeURL = URL;
+ vi.stubGlobal('URL', class extends NativeURL { static createObjectURL = create; static revokeObjectURL = revoke; });
+ vi.stubGlobal('Image', class {
+ naturalWidth = 640; naturalHeight = 360;
+ onload: (() => void) | null = null; onerror: (() => void) | null = null;
+ set src(value: string) { if (!value) return; const load = () => this.onload?.(); if (deferred) loads.push(load); else queueMicrotask(load); }
+ });
+ const header = new TextEncoder().encode(JSON.stringify({ seq: 1, device_width: 1280, device_height: 720 }));
+ const frame = new Uint8Array(4 + header.length + 1); frame.set([1, 1]);
+ new DataView(frame.buffer).setUint16(2, header.length, false); frame.set(header, 4);
+ return { draw, create, revoke, frame, loads };
+}
+
+beforeEach(() => {
+ MockSocket.instances = []; actions.isReady = true; actions.sendRequest.mockReset();
+ actions.sendRequest.mockResolvedValue({ success: true });
+ vi.stubGlobal('WebSocket', MockSocket);
+ vi.stubGlobal('ResizeObserver', class { observe() {} disconnect() {} });
+});
+afterEach(() => { vi.restoreAllMocks(); vi.unstubAllGlobals(); });
+
+describe('BrowserWorkspace session lifecycle', () => {
+ it('does not connect without a saved workflow or a browser node', () => {
+ const { rerender } = render(<BrowserWorkspace nodes={nodes} />);
+ expect(screen.getByText('Save this workflow to open its browser.')).toBeInTheDocument();
+ rerender(<BrowserWorkspace workflowId="wf" nodes={[]} />);
+ expect(MockSocket.instances).toHaveLength(0);
+ });
+ it('attaches immediately on open, watches idle sessions, and launches only on Start', async () => {
+ render(<BrowserWorkspace workflowId="wf" nodes={nodes} />);
+ const socket = MockSocket.instances[0];
+ expect(socket.url).toMatch(/\/ws\/browser$/);
+ act(() => { socket.open(); socket.message({ type: 'attached', running: false }); });
+ expect(socket.sent[0]).toMatchObject({ type: 'attach', target: { kind: 'node', workflow_id: 'wf', node_id: 'browser-1' }, visible: true });
+ expect(actions.sendRequest.mock.calls.some(([type]) => type === 'browser_session_open')).toBe(false);
+ fireEvent.click(screen.getByRole('button', { name: 'Start browser' }));
+ await waitFor(() => expect(actions.sendRequest).toHaveBeenCalledWith('browser_session_open', { workflow_id: 'wf', node_id: 'browser-1' }, 60_000));
+ });
+ it('sends hidden visibility and closes the old socket when the selected node changes', () => {
+ const { rerender, unmount } = render(<BrowserWorkspace workflowId="wf" nodes={[...nodes, { node_id: 'browser-2', label: 'Orders' }]} />);
+ const old = MockSocket.instances[0]; act(() => old.open());
+ rerender(<BrowserWorkspace workflowId="wf" nodes={[...nodes, { node_id: 'browser-2', label: 'Orders' }]} visible={false} />);
+ expect(old.sent).toContainEqual({ type: 'visibility', visible: false });
+ fireEvent.change(screen.getByRole('combobox', { name: 'Browser node' }), { target: { value: 'browser-2' } });
+ expect(old.close).toHaveBeenCalledOnce();
+ const current = MockSocket.instances[1]; act(() => current.open());
+ expect(current.sent[0]).toMatchObject({ target: { node_id: 'browser-2' }, visible: false });
+ unmount(); expect(current.close).toHaveBeenCalledOnce();
+ });
+ it('gates navigation and keyboard input on server-confirmed control and hands back on unmount', () => {
+ const { unmount } = render(<BrowserWorkspace workflowId="wf" nodes={nodes} />);
+ const socket = MockSocket.instances[0];
+ act(() => { socket.open(); socket.message({ type: 'state', state: 'agent', controller: null }); });
+ expect(screen.getByRole('textbox', { name: 'Browser address' })).toBeDisabled();
+ fireEvent.click(screen.getByRole('button', { name: 'Take control' }));
+ expect(socket.sent.at(-1)).toEqual({ type: 'control_request' });
+ act(() => socket.message({ type: 'state', state: 'user', controller: 'you' }));
+ expect(screen.getByRole('textbox', { name: 'Browser address' })).toBeEnabled();
+ fireEvent.keyDown(screen.getByRole('group', { name: 'Live browser page' }), { key: 'Enter', code: 'Enter', keyCode: 13 });
+ expect(socket.sent.at(-1)).toMatchObject({ type: 'key', action: 'down', key: 'Enter', key_code: 13 });
+ unmount(); expect(socket.sent.at(-1)).toEqual({ type: 'control_release' });
+ });
+ it('acknowledges malformed binary frames so a bad frame cannot stall the stream', async () => {
+ render(<BrowserWorkspace workflowId="wf" nodes={nodes} />);
+ const socket = MockSocket.instances[0];
+ act(() => { socket.open(); socket.onmessage?.({ data: new ArrayBuffer(2) }); });
+ await waitFor(() => expect(socket.sent.at(-1)).toEqual({ type: 'ack' }));
+ expect(screen.getByRole('alert')).toHaveTextContent('Incomplete browser frame');
+ });
+ it('paints and acknowledges frames, revokes JPEG URLs, and skips decoding hidden frames', async () => {
+ const { draw, create, revoke, frame } = mockFrameRenderer();
+ const { rerender } = render(<BrowserWorkspace workflowId="wf" nodes={nodes} />);
+ const socket = MockSocket.instances[0];
+ act(() => { socket.open(); socket.onmessage?.({ data: frame.buffer }); });
+ await waitFor(() => expect(draw).toHaveBeenCalledOnce());
+ expect(socket.sent.at(-1)).toEqual({ type: 'ack', seq: 1 });
+ expect(revoke).toHaveBeenCalledWith('blob:browser-frame');
+ rerender(<BrowserWorkspace workflowId="wf" nodes={nodes} visible={false} />);
+ act(() => socket.onmessage?.({ data: frame.buffer }));
+ await waitFor(() => expect(socket.sent.filter((message) => message.type === 'ack')).toHaveLength(2));
+ expect(create).toHaveBeenCalledOnce();
+ });
+ it('clears the picture and offers Reconnect after terminal screencast failure', async () => {
+ const { draw, frame } = mockFrameRenderer();
+ render(<BrowserWorkspace workflowId="wf" nodes={nodes} />);
+ const socket = MockSocket.instances[0];
+ act(() => { socket.open(); socket.onmessage?.({ data: frame.buffer }); });
+ await waitFor(() => expect(draw).toHaveBeenCalledOnce());
+ act(() => {
+ socket.message({ type: 'error', code: 'screencast', retrying: false, message: 'Reconnect the live view.' });
+ // A controller broadcast does not repair a failed picture.
+ socket.message({ type: 'state', state: 'idle', controller: null });
+ });
+ expect(screen.getByLabelText('Browser screenshot')).not.toBeVisible();
+ expect(screen.getByRole('alert')).toHaveTextContent('Reconnect the live view.');
+ fireEvent.click(screen.getByRole('button', { name: 'Reconnect' }));
+ expect(socket.close).toHaveBeenCalledOnce();
+ expect(MockSocket.instances).toHaveLength(2);
+ expect(screen.queryByRole('alert')).not.toBeInTheDocument();
+ });
+ it('recovers a retrying screencast error on a fresh frame without hiding navigation errors', async () => {
+ const { draw, frame } = mockFrameRenderer();
+ render(<BrowserWorkspace workflowId="wf" nodes={nodes} />);
+ const socket = MockSocket.instances[0];
+ act(() => {
+ socket.open(); socket.message({ type: 'state', state: 'agent' });
+ socket.message({ type: 'error', code: 'screencast', retrying: true, message: 'Retrying live view…' });
+ });
+ expect(screen.queryByRole('button', { name: 'Reconnect' })).not.toBeInTheDocument();
+ expect(screen.getByRole('status')).toHaveTextContent('Employee is browsing');
+ act(() => socket.onmessage?.({ data: frame.buffer }));
+ await waitFor(() => expect(draw).toHaveBeenCalledOnce());
+ expect(screen.queryByRole('alert')).not.toBeInTheDocument();
+ act(() => {
+ socket.message({ type: 'error', code: 'screencast', retrying: true, message: 'Retrying live view…' });
+ socket.message({ type: 'error', code: 'navigation', message: 'This address is blocked.' });
+ socket.onmessage?.({ data: frame.buffer });
+ });
+ await waitFor(() => expect(draw).toHaveBeenCalledTimes(2));
+ expect(screen.getByRole('alert')).toHaveTextContent('This address is blocked.');
+ });
+});
+
+
+describe('browser frame and input lifecycle boundaries', () => {
+ it('does not reset an unchanged canvas backing store', async () => {
+ const { frame, draw } = mockFrameRenderer();
+ const width = vi.spyOn(HTMLCanvasElement.prototype, 'width', 'set');
+ const height = vi.spyOn(HTMLCanvasElement.prototype, 'height', 'set');
+ render(<BrowserWorkspace workflowId="wf" nodes={nodes} />);
+ const socket = MockSocket.instances[0];
+ act(() => { socket.open(); socket.onmessage?.({ data: frame.buffer }); });
+ await waitFor(() => expect(draw).toHaveBeenCalledOnce());
+ act(() => socket.onmessage?.({ data: frame.buffer }));
+ await waitFor(() => expect(draw).toHaveBeenCalledTimes(2));
+ expect(width).toHaveBeenCalledTimes(1); expect(height).toHaveBeenCalledTimes(1);
+ });
+ it('does not paint after hiding during decode, but still acknowledges the frame', async () => {
+ const { frame, loads, draw, revoke } = mockFrameRenderer(true);
+ const { rerender } = render(<BrowserWorkspace workflowId="wf" nodes={nodes} />);
+ const socket = MockSocket.instances[0];
+ act(() => { socket.open(); socket.onmessage?.({ data: frame.buffer }); });
+ await waitFor(() => expect(loads).toHaveLength(1));
+ rerender(<BrowserWorkspace workflowId="wf" nodes={nodes} visible={false} />);
+ await act(async () => loads[0]());
+ expect(draw).not.toHaveBeenCalled(); expect(revoke).toHaveBeenCalled();
+ expect(socket.sent).toContainEqual({ type: 'ack', seq: 1 });
+ });
+ it('does not restore live state when an old decode finishes after disconnect', async () => {
+ const { frame, loads, draw, revoke } = mockFrameRenderer(true);
+ render(<BrowserWorkspace workflowId="wf" nodes={nodes} />);
+ const socket = MockSocket.instances[0];
+ act(() => { socket.open(); socket.onmessage?.({ data: frame.buffer }); });
+ await waitFor(() => expect(loads).toHaveLength(1));
+ act(() => { socket.readyState = 3; socket.onclose?.({ code: 4004, reason: 'Session closed' }); });
+ await act(async () => loads[0]());
+ expect(draw).not.toHaveBeenCalled(); expect(revoke).toHaveBeenCalled();
+ expect(screen.getByRole('status')).toHaveTextContent('Disconnected');
+ expect(screen.getByLabelText('Browser screenshot')).not.toBeVisible();
+ expect(screen.getByRole('button', { name: 'Reconnect' })).toBeInTheDocument();
+ });
+ it('cleans up an in-progress decode on disposal without painting', async () => {
+ const { frame, loads, draw, revoke } = mockFrameRenderer(true);
+ const { unmount } = render(<BrowserWorkspace workflowId="wf" nodes={nodes} />);
+ const socket = MockSocket.instances[0];
+ act(() => { socket.open(); socket.onmessage?.({ data: frame.buffer }); });
+ await waitFor(() => expect(loads).toHaveLength(1));
+ unmount(); await act(async () => loads[0]());
+ expect(draw).not.toHaveBeenCalled(); expect(revoke).toHaveBeenCalled();
+ expect(socket.close).toHaveBeenCalledOnce();
+ });
+ it.each(['blur', 'pointercancel'])('releases control on %s and stops further keys', (kind) => {
+ render(<BrowserWorkspace workflowId="wf" nodes={nodes} />);
+ const socket = MockSocket.instances[0];
+ act(() => { socket.open(); socket.message({ type: 'state', state: 'user', controller: 'you' }); });
+ const surface = screen.getByRole('group', { name: 'Live browser page' });
+ fireEvent.keyDown(surface, { key: 'Shift', code: 'ShiftLeft' });
+ if (kind === 'blur') fireEvent(window, new Event('blur'));
+ else fireEvent.pointerCancel(surface);
+ expect(socket.sent.at(-1)).toEqual({ type: 'control_release' });
+ const count = socket.sent.length;
+ fireEvent.keyDown(surface, { key: 'a', code: 'KeyA' });
+ expect(socket.sent).toHaveLength(count);
+ });
+ it('cancels a pending takeover when the document becomes hidden', () => {
+ render(<BrowserWorkspace workflowId="wf" nodes={nodes} />);
+ const socket = MockSocket.instances[0];
+ act(() => { socket.open(); socket.message({ type: 'state', state: 'agent' }); });
+ fireEvent.click(screen.getByRole('button', { name: 'Take control' }));
+ expect(socket.sent.at(-1)).toEqual({ type: 'control_request' });
+ vi.spyOn(document, 'visibilityState', 'get').mockReturnValue('hidden');
+ fireEvent(document, new Event('visibilitychange'));
+ expect(socket.sent.slice(-2)).toEqual([{ type: 'visibility', visible: false }, { type: 'control_release' }]);
+ expect(screen.getByRole('button', { name: 'Take control' })).toBeEnabled();
+ });
+ it('bounds decoding when a sender exceeds its two-frame credit', async () => {
+ const { frame, loads } = mockFrameRenderer(true);
+ render(<BrowserWorkspace workflowId="wf" nodes={nodes} />);
+ const socket = MockSocket.instances[0];
+ act(() => { socket.open(); for (let i = 0; i < 10; i++) socket.onmessage?.({ data: frame.buffer }); });
+ await waitFor(() => expect(loads).toHaveLength(1));
+ expect(socket.sent.filter((m) => m.type === 'ack')).toHaveLength(8);
+ await act(async () => loads[0]());
+ expect(loads).toHaveLength(2);
+ await act(async () => loads[1]());
+ expect(socket.sent.filter((m) => m.type === 'ack')).toHaveLength(10);
+ });
+});
diff --git a/client/src/components/browser/__tests__/protocol.test.ts b/client/src/components/browser/__tests__/protocol.test.ts
new file mode 100644
index 00000000..98ed9996
--- /dev/null
+++ b/client/src/components/browser/__tests__/protocol.test.ts
@@ -0,0 +1,37 @@
+import { describe, expect, it } from 'vitest';
+import { browserModifiers, browserPoint, decodeBrowserFrame } from '../protocol';
+
+const metadata = { seq: 7, device_width: 1280, device_height: 720, page_scale_factor: 1, offset_top: 0 };
+function envelope(header = metadata): ArrayBuffer {
+ const json = new TextEncoder().encode(JSON.stringify(header));
+ const result = new Uint8Array(4 + json.length + 3);
+ result.set([1, 1]); new DataView(result.buffer).setUint16(2, json.length, false);
+ result.set(json, 4); result.set([255, 216, 255], 4 + json.length);
+ return result.buffer;
+}
+
+describe('browser frame protocol', () => {
+ it('reads a big-endian JSON header and separates the JPEG bytes', () => {
+ const result = decodeBrowserFrame(envelope());
+ expect(result.header).toEqual(metadata);
+ expect(result.jpeg.size).toBe(3);
+ expect(result.jpeg.type).toBe('image/jpeg');
+ });
+ it('rejects unsupported versions, truncated headers and invalid dimensions', () => {
+ const version = envelope(); new Uint8Array(version)[0] = 2;
+ expect(() => decodeBrowserFrame(version)).toThrow('Unsupported');
+ expect(() => decodeBrowserFrame(envelope().slice(0, 7))).toThrow('Incomplete');
+ expect(() => decodeBrowserFrame(envelope({ ...metadata, device_width: 0 }))).toThrow('Invalid');
+ });
+ it('maps the center of a letterboxed image to viewport CSS pixels', () => {
+ expect(browserPoint(200, 200, 400, 400, 640, 360, metadata)).toEqual({ x: 640, y: 360 });
+ expect(browserPoint(200, 20, 400, 400, 640, 360, metadata)).toBeNull();
+ });
+ it('maps pillarboxing and adjusts for page scale and browser top offset', () => {
+ expect(browserPoint(400, 200, 800, 400, 640, 640, { ...metadata, device_height: 1280, page_scale_factor: 2, offset_top: 20 })).toEqual({ x: 320, y: 310 });
+ expect(browserPoint(10, 200, 800, 400, 640, 640, metadata)).toBeNull();
+ });
+ it('uses CDP modifier bits', () => {
+ expect(browserModifiers({ altKey: true, ctrlKey: true, metaKey: false, shiftKey: true })).toBe(11);
+ });
+});
diff --git a/client/src/components/browser/protocol.ts b/client/src/components/browser/protocol.ts
new file mode 100644
index 00000000..0163e9c3
--- /dev/null
+++ b/client/src/components/browser/protocol.ts
@@ -0,0 +1,45 @@
+/** Versioned browser stream envelope. The bytes after the UTF-8 JSON header are JPEG. */
+export interface BrowserFrameHeader {
+ seq: number;
+ target_id?: string;
+ device_width: number;
+ device_height: number;
+ page_scale_factor?: number;
+ offset_top?: number;
+}
+
+export function decodeBrowserFrame(buffer: ArrayBuffer): { header: BrowserFrameHeader; jpeg: Blob } {
+ if (buffer.byteLength < 5) throw new Error('Incomplete browser frame');
+ const view = new DataView(buffer);
+ if (view.getUint8(0) !== 1 || view.getUint8(1) !== 1) throw new Error('Unsupported browser stream version');
+ const end = 4 + view.getUint16(2, false);
+ if (end >= buffer.byteLength) throw new Error('Incomplete browser frame payload');
+ const header = JSON.parse(new TextDecoder().decode(new Uint8Array(buffer, 4, end - 4))) as BrowserFrameHeader;
+ if (!Number.isFinite(header.seq) || !Number.isFinite(header.device_width) || !Number.isFinite(header.device_height) || !(header.device_width > 0) || !(header.device_height > 0)) {
+ throw new Error('Invalid browser frame metadata');
+ }
+ return { header, jpeg: new Blob([new Uint8Array(buffer, end)], { type: 'image/jpeg' }) };
+}
+
+/** Map the aspect-fitted image into CDP viewport CSS coordinates; letterbox clicks are ignored. */
+export function browserPoint(
+ x: number, y: number, width: number, height: number,
+ imageWidth: number, imageHeight: number, header: BrowserFrameHeader,
+): { x: number; y: number } | null {
+ if (![width, height, imageWidth, imageHeight].every((n) => n > 0)) return null;
+ const scale = Math.min(width / imageWidth, height / imageHeight);
+ const drawnWidth = imageWidth * scale;
+ const drawnHeight = imageHeight * scale;
+ const px = x - (width - drawnWidth) / 2;
+ const py = y - (height - drawnHeight) / 2;
+ if (px < 0 || py < 0 || px > drawnWidth || py > drawnHeight) return null;
+ const pageScale = header.page_scale_factor && header.page_scale_factor > 0 ? header.page_scale_factor : 1;
+ return {
+ x: (px / drawnWidth * header.device_width) / pageScale,
+ y: Math.max(0, (py / drawnHeight * header.device_height - (header.offset_top ?? 0)) / pageScale),
+ };
+}
+
+export function browserModifiers(event: { altKey: boolean; ctrlKey: boolean; metaKey: boolean; shiftKey: boolean }): number {
+ return (event.altKey ? 1 : 0) | (event.ctrlKey ? 2 : 0) | (event.metaKey ? 4 : 0) | (event.shiftKey ? 8 : 0);
+}
diff --git a/client/src/components/ui/CanvasDock.tsx b/client/src/components/ui/CanvasDock.tsx
index be4ba020..28b95225 100644
--- a/client/src/components/ui/CanvasDock.tsx
+++ b/client/src/components/ui/CanvasDock.tsx
@@ -1,5 +1,5 @@
/**
- * Docked, resizable Canvas sidebar — the persistent right-hand viewing
+ * Docked, resizable workflow workspace — the persistent right-hand viewing
* surface (Claude-sidebar posture): stays open while working on the graph,
* auto-opens when content is pushed (canvas_updated -> notifyPushed), and
* doubles as the ephemeral click-to-preview surface for workspace files.
@@ -9,8 +9,8 @@
* - 'ephemeral': renders one transient item that lives in no board.
*/
-import React, { useMemo } from 'react';
-import { Monitor, PanelRightClose, Undo2 } from 'lucide-react';
+import React, { useMemo, useState, useSyncExternalStore } from 'react';
+import { Maximize2, Minimize2, Monitor, PanelRightClose, Undo2 } from 'lucide-react';
import type { Node } from 'reactflow';
import { toast } from 'sonner';
@@ -36,6 +36,9 @@ import {
useCanvasDockStore,
} from '../../stores/canvasDockStore';
import CanvasContent from '../parameterPanel/canvas/CanvasContent';
+import BrowserWorkspace from '../browser/BrowserWorkspace';
+import { WorkspaceTabs } from '../workspace/WorkspaceTabs';
+import { queryClient } from '../../lib/queryClient';
interface CanvasDockProps {
nodes: Node[];
@@ -49,6 +52,9 @@ const nodeLabel = (node: Node): string =>
const CanvasDock: React.FC<CanvasDockProps> = ({ nodes }) => {
const open = useCanvasDockStore((s) => s.open);
+ const tab = useCanvasDockStore((s) => s.tab);
+ const setTab = useCanvasDockStore((s) => s.setTab);
+ const [wide, setWide] = useState(false);
const widthPx = useCanvasDockStore((s) => s.widthPx);
const mode = useCanvasDockStore((s) => s.mode);
const selectedNodeId = useCanvasDockStore((s) => s.selectedNodeId);
@@ -62,7 +68,25 @@ const CanvasDock: React.FC<CanvasDockProps> = ({ nodes }) => {
const workflowId = useAppStore((s) => s.currentWorkflow?.id);
- const canvasNodes = useMemo(() => nodes.filter(isCanvasNode), [nodes]);
+ // Schema hydration can finish after the graph mounts. Subscribe to hints
+ // so newly available Browser/Canvas definitions appear without a graph edit.
+ const hintsVersion = useSyncExternalStore(
+ (notify) => queryClient.getQueryCache().subscribe((event) => {
+ if (event.query.queryKey[0] === 'nodeSpec') notify();
+ }),
+ () => nodes.map((node) => {
+ const hints = resolveNodeDescription(node.type || '')?.uiHints;
+ return `${node.id}:${!!hints?.isCanvasPanel}:${!!hints?.isBrowserPanel}`;
+ }).join('|'),
+ );
+ // eslint-disable-next-line react-hooks/exhaustive-deps -- hintsVersion tracks external schema hydration.
+ const canvasNodes = useMemo(() => nodes.filter(isCanvasNode), [nodes, hintsVersion]);
+ const browserNodes = useMemo(() => nodes
+ .filter((node) => resolveNodeDescription(node.type || '')?.uiHints?.isBrowserPanel === true)
+ .map((node) => ({ node_id: node.id, label: nodeLabel(node) })),
+ // eslint-disable-next-line react-hooks/exhaustive-deps -- hintsVersion tracks external schema hydration.
+ [nodes, hintsVersion]);
+ const shownWidth = wide ? Math.min(1100, window.innerWidth - 160) : Math.min(widthPx, window.innerWidth - 40);
// A stale selection (workflow switch, node deleted) falls back to the
// first Canvas node rather than a dead board.
@@ -73,7 +97,7 @@ const CanvasDock: React.FC<CanvasDockProps> = ({ nodes }) => {
return canvasNodes[0]?.id ?? null;
}, [canvasNodes, selectedNodeId]);
- const showBoard = open && mode === 'node';
+ const showBoard = open && tab === 'board' && mode === 'node';
const board = useCanvasBoardQuery(
showBoard ? workflowId : null,
showBoard ? effectiveNodeId : null,
@@ -89,11 +113,12 @@ const CanvasDock: React.FC<CanvasDockProps> = ({ nodes }) => {
// take nearly the whole window; the 160px remainder keeps the handle
// and a sliver of canvas reachable to drag it back.
const viewportCap = Math.min(DOCK_MAX_WIDTH, window.innerWidth - 160);
+ setWide(false);
setWidth(
Math.min(viewportCap, Math.max(DOCK_MIN_WIDTH, startValue - deltaPx)),
);
},
- getStartValue: () => useCanvasDockStore.getState().widthPx,
+ getStartValue: () => shownWidth,
});
return (
@@ -101,8 +126,11 @@ const CanvasDock: React.FC<CanvasDockProps> = ({ nodes }) => {
className={`relative flex h-full shrink-0 overflow-hidden bg-bg-panel ${
open ? 'border-l border-border-default' : ''
} ${resize.isResizing ? '[transition:none]' : 'transition-[width] duration-300'}`}
- style={{ width: open ? widthPx : 0 }}
+ style={{ width: open ? shownWidth : 0 }}
aria-hidden={!open}
+ inert={!open}
+ role="complementary"
+ aria-label="Workspace"
>
<div
className={`h-full w-1.5 shrink-0 cursor-ew-resize transition-colors ${
@@ -113,7 +141,7 @@ const CanvasDock: React.FC<CanvasDockProps> = ({ nodes }) => {
onMouseDown={resize.start}
role="separator"
aria-orientation="vertical"
- aria-label="Resize canvas panel"
+ aria-label="Resize workspace"
/>
{/* pointer-events-none while dragging: an embedded iframe (web/PDF
@@ -124,10 +152,20 @@ const CanvasDock: React.FC<CanvasDockProps> = ({ nodes }) => {
resize.isResizing ? 'pointer-events-none' : ''
}`}
>
- <div className="flex shrink-0 items-center gap-2 border-b border-border-default px-3 py-2">
+ <div className="flex h-12 shrink-0 items-center gap-2 border-b border-border-default px-3">
<Monitor className="h-4 w-4 shrink-0 text-fg-muted" />
- <span className="shrink-0 text-sm font-semibold text-fg-default">Canvas</span>
-
+ <span className="min-w-0 flex-1 text-sm font-semibold text-fg-default">Workspace</span>
+ <Button variant="ghost" size="icon-sm" onClick={() => setWide(!wide)} aria-label={wide ? 'Restore size' : 'Expand'}>
+ {wide ? <Minimize2 /> : <Maximize2 />}
+ </Button>
+ <Button variant="ghost" size="icon-sm" onClick={close} aria-label="Close workspace"><PanelRightClose /></Button>
+ </div>
+ {open && <WorkspaceTabs
+ tab={tab}
+ onTabChange={setTab}
+ browser={<BrowserWorkspace key={workflowId ?? 'unsaved'} workflowId={workflowId} nodes={browserNodes} visible={open && tab === 'browser'} />}
+ board={<>
+ <div className="mb-2 flex shrink-0 items-center gap-2">
{mode === 'ephemeral' ? (
<>
<Badge variant="secondary" className="shrink-0">Preview</Badge>
@@ -169,18 +207,9 @@ const CanvasDock: React.FC<CanvasDockProps> = ({ nodes }) => {
</span>
)}
- <Button
- variant="ghost"
- size="icon-sm"
- onClick={close}
- aria-label="Close canvas panel"
- className="shrink-0"
- >
- <PanelRightClose className="h-4 w-4" />
- </Button>
</div>
- <div className="flex min-h-0 flex-1 flex-col p-3">
+ <div className="flex min-h-0 min-w-0 flex-1 flex-col">
{mode === 'ephemeral' && ephemeralItem ? (
<CanvasContent items={[ephemeralItem]} workflowId={workflowId} />
) : !effectiveNodeId ? (
@@ -204,6 +233,8 @@ const CanvasDock: React.FC<CanvasDockProps> = ({ nodes }) => {
/>
)}
</div>
+ </>}
+ />}
</div>
</div>
);
diff --git a/client/src/components/ui/TopToolbar.tsx b/client/src/components/ui/TopToolbar.tsx
index 4b67831c..f1fd73c4 100644
--- a/client/src/components/ui/TopToolbar.tsx
+++ b/client/src/components/ui/TopToolbar.tsx
@@ -562,7 +562,8 @@ const TopToolbar: React.FC<TopToolbarProps> = ({
size="icon-sm"
onClick={toggleCanvasDock}
aria-pressed={canvasDockOpen}
- title={canvasDockOpen ? 'Hide canvas' : 'Show canvas'}
+ title={canvasDockOpen ? 'Hide workspace' : 'Show workspace'}
+ aria-label="Workspace"
className="border-action-tools-border bg-action-tools-soft text-action-tools-ink hover:bg-action-tools-hover aria-pressed:bg-action-tools-hover"
>
<Monitor />
diff --git a/client/src/components/ui/__tests__/WorkspaceDock.test.tsx b/client/src/components/ui/__tests__/WorkspaceDock.test.tsx
new file mode 100644
index 00000000..d7d32948
--- /dev/null
+++ b/client/src/components/ui/__tests__/WorkspaceDock.test.tsx
@@ -0,0 +1,64 @@
+import { act, fireEvent, render, screen } from '@testing-library/react';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import type { Node } from 'reactflow';
+
+const mocks = vi.hoisted(() => ({
+ workflow: { id: 'wf1' },
+ hints: {} as Record<string, { isCanvasPanel?: boolean; isBrowserPanel?: boolean }>,
+ listeners: new Set<(event: { query: { queryKey: string[] } }) => void>(),
+}));
+vi.mock('../../../store/useAppStore', () => ({ useAppStore: (select: (s: unknown) => unknown) => select({ currentWorkflow: mocks.workflow }) }));
+vi.mock('../../../lib/nodeSpec', () => ({ resolveNodeDescription: (type: string) => ({ uiHints: mocks.hints[type] }) }));
+vi.mock('../../../lib/queryClient', () => ({ queryClient: { getQueryCache: () => ({ subscribe: (fn: (event: { query: { queryKey: string[] } }) => void) => {
+ mocks.listeners.add(fn); return () => mocks.listeners.delete(fn);
+} }) } }));
+vi.mock('../../../hooks/useCanvasBoard', () => ({ useCanvasBoardQuery: () => ({ data: { items: [] } }), useCanvasRemove: () => ({ mutate: vi.fn() }) }));
+vi.mock('../../parameterPanel/canvas/CanvasContent', () => ({ default: () => <div>Saved canvas board</div> }));
+vi.mock('../../browser/BrowserWorkspace', () => ({ default: ({ workflowId, nodes }: { workflowId: string; nodes: { node_id: string; label: string }[] }) => (
+ <div data-testid="browser-view" data-workflow={workflowId}>{nodes.map((node) => <span key={node.node_id}>{node.label}</span>)}</div>
+) }));
+
+import CanvasDock from '../CanvasDock';
+import { useCanvasDockStore } from '../../../stores/canvasDockStore';
+
+const nodes: Node[] = [
+ { id: 'board1', type: 'boardPlugin', position: { x: 0, y: 0 }, data: { label: 'Board' } },
+ { id: 'web1', type: 'webPlugin', position: { x: 1, y: 0 }, data: { label: 'Research browser' } },
+];
+
+beforeEach(() => {
+ mocks.hints = { boardPlugin: { isCanvasPanel: true }, webPlugin: { isBrowserPanel: true } };
+ useCanvasDockStore.setState({ open: true, widthPx: 460, mode: 'node', tab: 'board', selectedNodeId: null, ephemeralItem: null });
+});
+
+describe('Developer workspace', () => {
+ it('shares Browser, Canvas and Android tabs and the workflow-bound viewer', () => {
+ render(<CanvasDock nodes={nodes} />);
+ expect(screen.getByText('Saved canvas board')).toBeInTheDocument();
+ fireEvent.mouseDown(screen.getByRole('tab', { name: 'Browser' }), { button: 0, ctrlKey: false });
+ expect(screen.getByText('Research browser')).toBeInTheDocument();
+ expect(screen.getByTestId('browser-view')).toHaveAttribute('data-workflow', 'wf1');
+ fireEvent.mouseDown(screen.getByRole('tab', { name: 'Android' }), { button: 0, ctrlKey: false });
+ expect(screen.getByText('The Android mirror isn’t available yet')).toBeInTheDocument();
+ expect(screen.queryByTestId('browser-view')).toBeNull();
+ });
+
+ it('discovers a browser after schema hydration without a graph edit', () => {
+ mocks.hints.webPlugin = {};
+ useCanvasDockStore.setState({ tab: 'browser' });
+ render(<CanvasDock nodes={nodes} />);
+ expect(screen.queryByText('Research browser')).toBeNull();
+ act(() => {
+ mocks.hints.webPlugin = { isBrowserPanel: true };
+ for (const fn of mocks.listeners) fn({ query: { queryKey: ['nodeSpec'] } });
+ });
+ expect(screen.getByText('Research browser')).toBeInTheDocument();
+ });
+
+ it('unmounts the live viewer when closed', () => {
+ useCanvasDockStore.setState({ tab: 'browser' });
+ render(<CanvasDock nodes={nodes} />);
+ fireEvent.click(screen.getByRole('button', { name: 'Close workspace' }));
+ expect(screen.queryByTestId('browser-view')).toBeNull();
+ });
+});
diff --git a/client/src/components/workspace/WorkspaceTabs.tsx b/client/src/components/workspace/WorkspaceTabs.tsx
new file mode 100644
index 00000000..9e524ff8
--- /dev/null
+++ b/client/src/components/workspace/WorkspaceTabs.tsx
@@ -0,0 +1,47 @@
+import { Globe, PanelsTopLeft, Smartphone } from 'lucide-react';
+import { Tabs as TabsPrimitive } from 'radix-ui';
+import type { ReactNode } from 'react';
+
+export type WorkspaceTab = 'browser' | 'board' | 'android';
+
+/** Shared navigation and sizing contract for Home and the workflow editor. */
+export function WorkspaceTabs({ tab, onTabChange, browser, board, android }: {
+ tab: WorkspaceTab;
+ onTabChange: (tab: WorkspaceTab) => void;
+ browser: ReactNode;
+ board: ReactNode;
+ android?: ReactNode;
+}) {
+ return (
+ <TabsPrimitive.Root value={tab} onValueChange={(value) => onTabChange(value as WorkspaceTab)} className="flex min-h-0 min-w-0 flex-1 flex-col">
+ <TabsPrimitive.List aria-label="Workspace views" className="flex h-10 shrink-0 gap-0.5 overflow-x-auto border-b border-border-default px-2">
+ {([
+ ['browser', 'Browser', Globe],
+ ['board', 'Canvas', PanelsTopLeft],
+ ['android', 'Android', Smartphone],
+ ] as const).map(([value, label, Icon]) => (
+ <TabsPrimitive.Trigger key={value} value={value} className="flex h-10 items-center gap-1.75 whitespace-nowrap border-b-2 border-transparent px-2.5 text-sm font-medium text-fg-muted outline-none transition-colors hover:text-fg-default focus-visible:ring-3 focus-visible:ring-ring/50 data-[state=active]:border-node-agent data-[state=active]:text-fg-default">
+ <Icon aria-hidden className="size-3.75" strokeWidth={1.75} />{label}
+ </TabsPrimitive.Trigger>
+ ))}
+ </TabsPrimitive.List>
+ {([
+ ['browser', browser], ['board', board], ['android', android ?? <AndroidWorkspace />],
+ ] as const).map(([value, content]) => (
+ <TabsPrimitive.Content key={value} value={value} className="flex min-h-0 min-w-0 flex-1 flex-col overflow-hidden bg-bg-app p-3 outline-none">
+ {content}
+ </TabsPrimitive.Content>
+ ))}
+ </TabsPrimitive.Root>
+ );
+}
+
+export function AndroidWorkspace() {
+ return (
+ <div className="m-auto flex max-w-80 flex-col items-center gap-2 p-6 text-center">
+ <Smartphone aria-hidden className="size-6 text-fg-faint" strokeWidth={1.5} />
+ <p className="m-0 text-base font-medium text-fg-default">The Android mirror isn’t available yet</p>
+ <p className="m-0 text-sm text-fg-muted">Android actions can run in your workflow. A live device view is not available yet.</p>
+ </div>
+ );
+}
diff --git a/client/src/features/home/__tests__/workspace.test.tsx b/client/src/features/home/__tests__/workspace.test.tsx
index 43ecb82b..162be63c 100644
--- a/client/src/features/home/__tests__/workspace.test.tsx
+++ b/client/src/features/home/__tests__/workspace.test.tsx
@@ -19,6 +19,13 @@ vi.mock('@/contexts/WebSocketContext', async (importOriginal) => ({
}));
vi.mock('../../../app/useShellActions', () => ({ enterDev: vi.fn() }));
+vi.mock('@/components/browser/BrowserWorkspace', () => ({
+ default: ({ workflowId, nodes, visible }: { workflowId: string; nodes: { node_id: string; label: string }[]; visible: boolean }) => (
+ <div data-testid="browser-workspace" data-workflow={workflowId} data-visible={String(visible)}>
+ {nodes.length ? nodes.map((node) => <span key={node.node_id}>{node.label}</span>) : 'No Browser node in this workflow'}
+ </div>
+ ),
+}));
import { normalizeWorkflowControlStatus } from '@/contexts/WebSocketContext';
import { ThemeProvider } from '@/contexts/ThemeContext';
@@ -154,13 +161,24 @@ describe('WorkspaceDock', () => {
expect(screen.getByText('Leo’s workspace')).toBeInTheDocument();
});
- it('says plainly that Browser and Android are not built yet', async () => {
+ it('mounts the selected employee’s browser nodes and keeps the Android panel available', async () => {
useHomeStore.setState({ workspaceOpen: true, workspaceTab: 'browser' });
- renderWith([employee()]);
- expect(screen.getByText('The live browser view isn’t available yet')).toBeInTheDocument();
+ renderWith([employee({ browser_nodes: [{ node_id: 'w1:browser:1', label: 'Research browser' }] })]);
+ expect(screen.getByText('Research browser')).toBeInTheDocument();
+ expect(screen.getByTestId('browser-workspace')).toHaveAttribute('data-workflow', 'w1');
+ expect(screen.getByTestId('browser-workspace')).toHaveAttribute('data-visible', 'true');
await userEvent.click(screen.getByRole('tab', { name: 'Android' }));
expect(screen.getByText('The Android mirror isn’t available yet')).toBeInTheDocument();
expect(useHomeStore.getState().workspaceTab).toBe('android');
+ expect(screen.queryByTestId('browser-workspace')).toBeNull();
+ });
+
+ it('hides the live viewer when the workspace closes without changing its employee', () => {
+ useHomeStore.setState({ workspaceOpen: true, workspaceTab: 'browser' });
+ renderWith([employee()]);
+ act(() => useHomeStore.getState().closeWorkspace());
+ expect(screen.getByTestId('browser-workspace')).toHaveAttribute('data-visible', 'false');
+ expect(screen.getByTestId('browser-workspace')).toHaveAttribute('data-workflow', 'w1');
});
it('asks for no board when the employee has no Canvas', () => {
diff --git a/client/src/features/home/data/schemas.ts b/client/src/features/home/data/schemas.ts
index f1b54683..a6ce1aa0 100644
--- a/client/src/features/home/data/schemas.ts
+++ b/client/src/features/home/data/schemas.ts
@@ -53,6 +53,8 @@ export const employeeSummarySchema = z.object({
watch_node_ids: z.array(z.string()).catch([]),
/** The Canvas board the Workspace shows; null when the graph has none. */
canvas_node_id: z.string().nullable().catch(null),
+ /** Saved Browser nodes available in this employee's workspace. */
+ browser_nodes: z.array(z.object({ node_id: z.string().min(1), label: z.string() })).catch([]),
revision: z.number().catch(0),
hired_at: z.string().nullable().catch(null),
});
diff --git a/client/src/features/home/workspace/WorkspaceDock.tsx b/client/src/features/home/workspace/WorkspaceDock.tsx
index 1e36c520..857717ac 100644
--- a/client/src/features/home/workspace/WorkspaceDock.tsx
+++ b/client/src/features/home/workspace/WorkspaceDock.tsx
@@ -1,9 +1,8 @@
/**
* The Workspace (design handoff "Workspace panel"): a dock on the right of
* Home showing what one employee is working on. Canvas shows the
- * employee's Canvas board through the renderer the editor's Canvas panel
- * uses; Browser and Android say plainly that their live views are not
- * built yet.
+ * employee's board; Browser attaches to their saved browser nodes. The
+ * editor shares the same tabs and live browser surface.
*
* At 1100px and wider the dock pushes the page aside; narrower, it lies
* over the page with a shadow. The left edge drags from 360px to the
@@ -15,8 +14,9 @@
*/
import type { LucideIcon } from 'lucide-react';
-import { Code, Globe, Maximize2, Minimize2, Monitor, PanelsTopLeft, Smartphone, X } from 'lucide-react';
-import { Tabs as TabsPrimitive } from 'radix-ui';
+import { Code, Maximize2, Minimize2, Monitor, PanelsTopLeft, X } from 'lucide-react';
+import BrowserWorkspace from '@/components/browser/BrowserWorkspace';
+import { WorkspaceTabs } from '@/components/workspace/WorkspaceTabs';
import { Suspense, lazy, useCallback, useLayoutEffect, useRef, useState, type ReactNode } from 'react';
import { Button } from '@/components/ui/button';
import { Skeleton } from '@/components/ui/skeleton';
@@ -28,7 +28,7 @@ import { useEmployeesQuery } from '../data/employees';
import { useLiveTask } from '../data/liveTask';
import { presentEmployee } from '../data/presentation';
import type { EmployeeSummary } from '../data/schemas';
-import { useHomeStore, type WorkspaceTab } from '../state/homeStore';
+import { useHomeStore } from '../state/homeStore';
import { Avatar, StatusPill } from '../ui/primitives';
// Its own chunk: the board's viewers (markdown, code, JSON) stay out of Home's.
@@ -39,15 +39,6 @@ function dockWidth(widthPx: number, wide: boolean): string {
return wide ? 'max(420px, min(1100px, 100vw - 420px))' : `min(${widthPx}px, max(340px, 100vw - 40px))`;
}
-const TABS: { tab: WorkspaceTab; label: string; icon: LucideIcon }[] = [
- { tab: 'browser', label: 'Browser', icon: Globe },
- { tab: 'board', label: 'Canvas', icon: PanelsTopLeft },
- { tab: 'android', label: 'Android', icon: Smartphone },
-];
-
-const TAB_TRIGGER =
- 'flex h-10 items-center gap-1.75 border-b-2 border-transparent px-2.5 text-sm font-medium whitespace-nowrap text-fg-muted outline-none transition-colors hover:text-fg-default focus-visible:ring-3 focus-visible:ring-ring/50 data-[state=active]:border-node-agent data-[state=active]:text-fg-default';
-
const PANEL = 'flex min-h-0 flex-1 flex-col bg-bg-app p-3 outline-none';
function Note({ icon: Icon, title, detail, children }: { icon: LucideIcon; title: string; detail: string; children?: ReactNode }) {
@@ -120,54 +111,16 @@ function CanvasTab({ employee }: { employee: EmployeeSummary }) {
);
}
-function DockTabs({ employee }: { employee: EmployeeSummary }) {
+function DockTabs({ employee, visible }: { employee: EmployeeSummary; visible: boolean }) {
const tab = useHomeStore((s) => s.workspaceTab);
const setTab = useHomeStore((s) => s.setWorkspaceTab);
- // Only the active panel is mounted, so the shared ref is always it.
- const panelRef = useRef<HTMLDivElement>(null);
- const shownTab = useRef(tab);
- // A new tab's panel rises out of a blur (design handoff "Tabs").
- useLayoutEffect(() => {
- if (shownTab.current === tab) return;
- shownTab.current = tab;
- animate(
- panelRef.current,
- [
- { opacity: 0, transform: 'translateY(6px)', filter: 'blur(3px)' },
- { opacity: 1, transform: 'none', filter: 'blur(0)' },
- ],
- { duration: 'slow', easing: 'spring' },
- );
- }, [tab]);
-
return (
- <TabsPrimitive.Root value={tab} onValueChange={(next) => setTab(next as WorkspaceTab)} className="flex min-h-0 flex-1 flex-col">
- <TabsPrimitive.List aria-label="Views" className="flex h-10 shrink-0 gap-0.5 overflow-x-auto border-b border-border-default px-2">
- {TABS.map(({ tab: value, label, icon: Icon }) => (
- <TabsPrimitive.Trigger key={value} value={value} className={TAB_TRIGGER}>
- <Icon aria-hidden className="size-3.75" strokeWidth={1.75} />
- {label}
- </TabsPrimitive.Trigger>
- ))}
- </TabsPrimitive.List>
- <TabsPrimitive.Content ref={panelRef} value="browser" className={PANEL}>
- <Note
- icon={Globe}
- title="The live browser view isn’t available yet"
- detail={`When ${employee.name} works in a browser, you’ll be able to watch here.`}
- />
- </TabsPrimitive.Content>
- <TabsPrimitive.Content ref={panelRef} value="board" className={PANEL}>
- <CanvasTab employee={employee} />
- </TabsPrimitive.Content>
- <TabsPrimitive.Content ref={panelRef} value="android" className={PANEL}>
- <Note
- icon={Smartphone}
- title="The Android mirror isn’t available yet"
- detail={`When ${employee.name} uses your phone, you’ll be able to watch here.`}
- />
- </TabsPrimitive.Content>
- </TabsPrimitive.Root>
+ <WorkspaceTabs
+ tab={tab}
+ onTabChange={setTab}
+ browser={<BrowserWorkspace key={employee.workflow_id} workflowId={employee.workflow_id} nodes={employee.browser_nodes} visible={visible && tab === 'browser'} />}
+ board={<CanvasTab employee={employee} />}
+ />
);
}
@@ -277,7 +230,7 @@ export function WorkspaceDock() {
)}
<DockButtons />
</header>
- {employee ? <DockTabs employee={employee} /> : <NoEmployee status={team.status} />}
+ {employee ? <DockTabs employee={employee} visible={open} /> : <NoEmployee status={team.status} />}
</div>
)}
</aside>
diff --git a/client/src/stores/__tests__/canvasDockStore.test.ts b/client/src/stores/__tests__/canvasDockStore.test.ts
index 4ef42ae7..e404f670 100644
--- a/client/src/stores/__tests__/canvasDockStore.test.ts
+++ b/client/src/stores/__tests__/canvasDockStore.test.ts
@@ -21,6 +21,7 @@ const resetStore = () =>
widthPx: 380,
autoOpen: true,
followMode: false,
+ tab: 'board',
mode: 'node',
selectedNodeId: null,
ephemeralItem: null,
@@ -60,6 +61,12 @@ describe('canvasDockStore.notifyPushed', () => {
expect(state.mode).toBe('ephemeral');
expect(state.ephemeralItem).toEqual(ephemeral);
});
+
+ it('does not switch away from a live browser on a Canvas push', () => {
+ useCanvasDockStore.setState({ open: true, tab: 'browser' });
+ useCanvasDockStore.getState().notifyPushed('canvas-2');
+ expect(useCanvasDockStore.getState().tab).toBe('browser');
+ });
});
describe('canvasDockStore prefs', () => {
@@ -97,9 +104,11 @@ describe('canvasDockStore prefs', () => {
});
it('showEphemeral opens the dock and backToNode returns to the board', () => {
+ useCanvasDockStore.getState().setTab('browser');
useCanvasDockStore.getState().showEphemeral(ephemeral);
expect(useCanvasDockStore.getState().open).toBe(true);
expect(useCanvasDockStore.getState().mode).toBe('ephemeral');
+ expect(useCanvasDockStore.getState().tab).toBe('board');
useCanvasDockStore.getState().backToNode();
expect(useCanvasDockStore.getState().mode).toBe('node');
diff --git a/client/src/stores/canvasDockStore.ts b/client/src/stores/canvasDockStore.ts
index 5f077907..f6e93124 100644
--- a/client/src/stores/canvasDockStore.ts
+++ b/client/src/stores/canvasDockStore.ts
@@ -16,12 +16,14 @@ import { z } from 'zod';
import { create } from 'zustand';
import type { CanvasItem } from '../lib/canvasBoard';
+import type { WorkspaceTab } from '../components/workspace/WorkspaceTabs';
const dockPrefsSchema = z.object({
open: z.boolean().default(false),
widthPx: z.number().min(280).max(4000).default(380),
autoOpen: z.boolean().default(true),
followMode: z.boolean().default(false),
+ tab: z.enum(['browser', 'board', 'android']).catch('board'),
});
type CanvasDockPrefs = z.infer<typeof dockPrefsSchema>;
const DOCK_PREFS_KEY = 'canvas_dock_prefs_v1';
@@ -54,6 +56,8 @@ interface CanvasDockState {
widthPx: number;
autoOpen: boolean;
followMode: boolean;
+ tab: WorkspaceTab;
+ setTab: (tab: WorkspaceTab) => void;
/** 'node' renders a Canvas node's board; 'ephemeral' a transient preview. */
mode: 'node' | 'ephemeral';
selectedNodeId: string | null;
@@ -87,6 +91,7 @@ const persist = (state: CanvasDockState) =>
widthPx: state.widthPx,
autoOpen: state.autoOpen,
followMode: state.followMode,
+ tab: state.tab,
});
export const useCanvasDockStore = create<CanvasDockState>((set, get) => ({
@@ -94,6 +99,11 @@ export const useCanvasDockStore = create<CanvasDockState>((set, get) => ({
widthPx: persisted.widthPx,
autoOpen: persisted.autoOpen,
followMode: persisted.followMode,
+ tab: persisted.tab,
+ setTab: (tab) => {
+ set({ tab });
+ persist(get());
+ },
mode: 'node',
selectedNodeId: null,
ephemeralItem: null,
@@ -120,9 +130,9 @@ export const useCanvasDockStore = create<CanvasDockState>((set, get) => ({
persist(get());
},
selectNode: (nodeId) =>
- set({ mode: 'node', selectedNodeId: nodeId, ephemeralItem: null }),
+ set({ mode: 'node', selectedNodeId: nodeId, ephemeralItem: null, tab: 'board' }),
showEphemeral: (item) => {
- set({ mode: 'ephemeral', ephemeralItem: item, open: true });
+ set({ mode: 'ephemeral', ephemeralItem: item, open: true, tab: 'board' });
persist(get());
},
backToNode: () => set({ mode: 'node', ephemeralItem: null }),
@@ -132,7 +142,7 @@ export const useCanvasDockStore = create<CanvasDockState>((set, get) => ({
if (state.mode === 'ephemeral') return;
if (!state.open) {
if (!state.autoOpen) return;
- set({ open: true, mode: 'node', selectedNodeId: nodeId });
+ set({ open: true, mode: 'node', selectedNodeId: nodeId, tab: 'board' });
persist(get());
return;
}
diff --git a/client/src/types/INodeProperties.ts b/client/src/types/INodeProperties.ts
index bf0b7c87..ec1fae90 100644
--- a/client/src/types/INodeProperties.ts
+++ b/client/src/types/INodeProperties.ts
@@ -244,6 +244,8 @@ export interface INodeUIHints {
/** MiddleSection: render the pushed-content Canvas board. The docked
* canvas sidebar reads the same flag to find Canvas nodes. */
isCanvasPanel?: boolean;
+ /** Live browser view backed by the plugin's browser-session transport. */
+ isBrowserPanel?: boolean;
/** Special-case panel for gmaps_create with map preview. */
showLocationPanel?: boolean;
/** ConsolePanel: this node is a chat-message target. */
diff --git a/docs-internal/browser_workspace.md b/docs-internal/browser_workspace.md
new file mode 100644
index 00000000..7e0f27b6
--- /dev/null
+++ b/docs-internal/browser_workspace.md
@@ -0,0 +1,176 @@
+# Browser workspace
+
+Normal mode and Dev mode share `components/workspace/WorkspaceTabs.tsx`
+(Browser, Canvas, Android) and `components/browser/BrowserWorkspace.tsx`.
+The Dev toolbar's Workspace button opens the existing resizable dock;
+Canvas boards, push notifications and temporary file previews still use
+`canvasDockStore`. Its selected tab is persisted. A Canvas push does not
+switch an already-open Browser tab; explicitly opening a file preview does
+switch to Canvas. Android currently has an explanatory panel, not a live mirror.
+
+## Discovery and identity
+
+Normal-mode employee summaries include `browser_nodes: [{node_id, label}]`.
+`services/employees/graph_index.py` discovers nodes by the registered plugin's
+`isBrowserPanel` hint, including disabled nodes for manual viewing. Dev uses
+the same NodeSpec hint from the current graph and observes schema hydration.
+The viewer selects one node and keys its session by workflow and node ID.
+Viewing a saved node attaches a session but does not launch Chrome; Start
+browser calls the authorized `browser_session_open` handler. Unsaved workflows
+must be saved first. Adding a node to an existing workflow must also be saved
+before the server can authorize it.
+
+## Transport and display
+
+The dedicated same-origin `/ws/browser` socket uses the application's auth
+and Origin checks. The first message attaches a node target and reports the
+visible surface's width, height and DPR. Binary frames contain version/kind,
+a big-endian uint16 JSON-header length, JSON metadata, and JPEG bytes.
+`protocol.ts` validates the envelope. The viewer draws frames serially and
+acknowledges each consumed or skipped frame so the server's two-frame window
+does not stall. Object URLs are released after decoding and on cleanup.
+
+Frames fit the available surface without cropping. Input coordinates exclude
+letterboxing and map through the frame metadata to CDP page coordinates.
+ResizeObserver reports panel resizing; document visibility and dock visibility
+pause delivery. Switching tabs/nodes or modes releases the socket and input
+control. A hidden Home dock also releases control and stops visible streaming.
+
+Navigation, page-tab selection, mouse, wheel, keyboard, pasted text and page
+dialogs require Take control; the server remains the authority on the control
+lease. Hand back returns control to the agent. Another controlling viewer is
+identified explicitly before a forced takeover is offered.
+
+Transient socket disconnects reconnect. Authentication and target errors need
+explicit action. Initial CDP screencast failures produce visible errors and
+retry after 1, 2 and 4 seconds while a viewer is visible. After that, resizing,
+visibility changes or reconnecting can attempt attachment again.
+
+## Live-view latency and diagnostics
+
+The browser socket's receive loop handles frame ACKs without awaiting page
+commands. Input commands use a bounded scheduler; pointer moves and wheel
+updates coalesce, while click/key transitions retain ordering. Control release,
+disconnect and page changes invalidate pending input. Input has its own CDP
+session so resizing the screencast does not detach an active input command.
+
+Each viewer retains the newest unsent frame when its FPS limit is reached,
+then sends it when the limit allows. The final update must be delivered even
+when the page stops producing frames. Viewer ACKs identify an outstanding
+sequence number; duplicate or unknown ACKs do not open additional credit.
+Chrome screencast ACKs are tracked per event and per capture session, rather
+than replacing an earlier unacknowledged event with the newest one. The hub
+acknowledges each capture after admitting it to bounded viewer slots; it does
+not wait for client paint or a flush timer before returning Chrome's capture
+credit. The separate two-frame viewer window still bounds client delivery.
+
+Set `OPENCOMPANY_BROWSER_DIAGNOSTICS=1` before starting the server to enable
+bounded timing samples and counts (`nodes/browser/_stream_metrics.py`).
+Snapshots include p50/p95 timing values and counts, with at most 256 samples
+per timing name. For frontend decoding/drawing diagnostics, append
+`?browserStreamDebug=1` to the app URL and inspect the browser console.
+These diagnostics contain numeric timing/count information, not page content,
+typed text, credentials or screenshots. Drawing duration alone is not
+input-to-paint latency.
+
+### Reproducible local benchmark
+
+`server/scripts/benchmark_browser_stream.py` launches an **already installed**
+Chrome in a disposable profile, serves a controlled page on loopback, and
+executes the real `browser_live_view`, `Viewer`, `ScreencastHub`, WebSocket and
+CDP input paths. It never downloads Chrome or attaches to a user's profile.
+Authentication and profile/control lookup are process-local fixtures; it does
+not open the application database. External DNS resolution is blocked for the
+isolated browser. Chrome and the temporary server are stopped afterward.
+
+The page paints a binary counter for each injected mouse press. A small
+reference viewer decodes the streamed JPEG, reads that counter, draws it on a
+canvas, and measures through two `requestAnimationFrame` callbacks. This is a
+**real transport measurement with a browser paint proxy**, not physical display
+latency and not the production React renderer. Frame age uses Chrome's capture
+timestamp and the local viewer's clock. A sample that is not painted within
+the deadline is reported as a missed update; percentiles cover only observed
+updates and must always be read alongside the miss count. The nominal target
+is p95 at or below 200 ms with zero missed updates.
+
+From `server/` in PowerShell, preserve the baseline **before** changing stream
+code, then use matching settings for both measurements:
+
+```powershell
+$browserBaseline = Join-Path $env:TEMP 'opencompany-stream-baseline.py'
+Copy-Item -LiteralPath nodes/browser/_stream.py -Destination $browserBaseline
+.venv/Scripts/python scripts/benchmark_browser_stream.py --stream-source $browserBaseline --label baseline --samples 50 --warmup 5 --output "$env:TEMP/browser-before.json"
+# Apply the implementation change, then run:
+.venv/Scripts/python scripts/benchmark_browser_stream.py --label current --samples 50 --warmup 5 --output "$env:TEMP/browser-after.json"
+```
+
+Use `.venv/bin/python` on Linux/macOS. Supply `--chrome /path/to/chrome` if
+automatic discovery cannot find an installed browser. `--fps 10 --samples 20
+--warmup 3` exercises explicit frame throttling; keep all other settings equal.
+Reports record Chrome version, loaded stream-source hash, benchmark-script hash,
+configuration, raw timing samples, observed page counters, target-side click
+count and missed updates. Keep the complete source revision with the reports:
+`--stream-source` replaces only the stream module, while its supporting modules
+come from the current checkout.
+
+Run comparisons sequentially under comparable machine load; alternate before
+and after runs and repeat them before drawing performance conclusions. This
+fixture does not validate WAN latency, large third-party pages, the browser-use
+CLI/agent path, production ownership checks or all human-control races. Those
+remain separate correctness and end-to-end checks.
+
+### Local measurements, 2026-09-26
+
+The final paired runs used installed **Chrome 153.0.8010.53**, not the runtime's
+pinned Chrome 154. Chrome 153's live `/json/protocol` did not advertise
+`Page.startScreencast.sendLastFrame`; the implementation requests that optional
+field, but these results do not establish its behavior on pinned Chrome 154.
+The reference viewer reported `document.visibilityState === "visible"`.
+Runs were sequential, baseline then current, on the same host with a 1,000 ms
+sample deadline and 25–31 ms pauses between samples. The 30 fps runs used 50
+samples after five warmups; 10 fps used 20 samples after three warmups.
+
+| FPS | Stream | Input-to-paint p50 / p95 (ms) | Frame-age p50 / p95 (ms) | Missed by deadline |
+| --- | --- | --- | --- | --- |
+| 30 | Baseline | 30.4 / 40.1 | 15.05 / 18.81 | 1 / 50 |
+| 30 | Final | 29.6 / 41.8 | 16.10 / 22.64 | 0 / 50 |
+| 10 | Baseline | 36.7 / 49.4 | 20.10 / 30.66 | 10 / 20 |
+| 10 | Final | 80.8 / 129.9 | 65.84 / 87.18 | 0 / 20 |
+
+All target-side presses arrived in each run: 55/55 at 30 fps and 23/23 at
+10 fps, including warmups. No transport errors were recorded. Final reference
+decode/draw p95 was 7.1 ms at 30 fps and 8.2 ms at 10 fps; draw-to-double-rAF
+p95 was 5.7 ms in both. The final runs met the reference target of p95 below
+200 ms with zero misses. This is evidence of better delivery in this fixture,
+not a general speedup: baseline percentiles exclude missed updates, and the
+final 30 fps p95 was slightly higher.
+
+During development an intermediate implementation measured p95 265.8 ms and
+one missed deadline at 30 fps, although that counter eventually appeared.
+Earlier pilot and intermediate runs also varied substantially. The table
+measures the completed capture-credit change; it does not justify a production
+latency guarantee. Repeat alternating comparisons and test the production
+React renderer and pinned browser before making that claim.
+
+The four raw JSON reports were emitted with labels `baseline-paired`,
+`current-paired`, `baseline-paired-fps10`, and `current-paired-fps10`. The
+benchmark instructions above reproduce their settings. Recorded SHA-256s:
+
+```text
+baseline _stream.py: f290448fbe696bbe362a2b7343a3cb977556e3d28dc350997945e4ccd3fb14b3
+final _stream.py: 78d4f948a8cce3b0f8a2f5a14390f03c3bea76802f2c445955a835cd7f2f700c
+benchmark script: 283e28a5afa86c16370e39b989b8d800462b940d1e81f6755f29bded129a7716
+```
+
+## Tests
+
+- `client/src/components/browser/__tests__`: stream envelope, geometry, attach,
+ rendering/ACK, cleanup, visibility and control gating.
+- `client/src/features/home/__tests__/workspace.test.tsx`: employee identity and
+ browser discovery integration, tab changes and hidden dock state.
+- `client/src/components/ui/__tests__/WorkspaceDock.test.tsx`: Dev tabs, schema
+ hydration and viewer unmount on close.
+- `server/tests/nodes/browser/test_browser_stream.py`: fake-CDP startup failure
+ recovery and cancellation.
+- `server/tests/services/employees/test_list_employees.py`: capability-based
+ browser discovery in employee summaries.
diff --git a/docs-internal/canvas_node.md b/docs-internal/canvas_node.md
index 835e6ca0..2cad2485 100644
--- a/docs-internal/canvas_node.md
+++ b/docs-internal/canvas_node.md
@@ -9,6 +9,11 @@ open while working on the graph, auto-opens when content is pushed, and
doubles as the click-to-preview surface for workspace files, and the Canvas
tab of Home's Workspace dock.
+The Dev dock now shares Home's Browser / Canvas / Android workspace tabs.
+Canvas retains its board and file-preview behavior; live Chrome sessions use
+the separate [Browser workspace](./browser_workspace.md) surface. URL items
+on a Canvas still render as sandboxed iframe previews, not browser sessions.
+
Everything here composes existing patterns; the doc records which one each
piece copies and the few places where a *new* decision had to be made (the
iframe sandbox matrix, the PDF inline exception, the first client-side
diff --git a/docs-internal/normal_mode.md b/docs-internal/normal_mode.md
index 11511029..c29e2c50 100644
--- a/docs-internal/normal_mode.md
+++ b/docs-internal/normal_mode.md
@@ -119,7 +119,14 @@ the team.
JSON viewers out of Home's, and refreshes on `canvas_updated` like the
editor's hosts. An employee without a Canvas gets a note and Open
workflow.
-- **Browser** and **Android** say that their live views are not built yet.
+- **Browser**: the shared `components/browser/BrowserWorkspace` attaches to
+ the employee's saved Browser nodes, supplied by `browser_nodes` in its
+ summary. Multiple nodes get a selector. Running sessions appear automatically;
+ Start browser opens an idle session. The view supports navigation, tabs,
+ Take control / Hand back, and browser dialogs. Frames use `/ws/browser`,
+ not a URL iframe. See [Browser workspace](./browser_workspace.md).
+- **Android**: a shared panel explains that live mirroring is not yet available.
+ Dev mode uses the same three workspace tabs and browser viewer.
- **Size and motion**: 460px wide by default. The left edge drags from
360px to the window less 420px, and Expand gives a bigger dock without
a drag. At 1100px and wider the dock pushes the page aside; narrower, it
@@ -370,8 +377,8 @@ history). Client: `features/home/**/__tests__`, `app/__tests__`,
library is shared across users in multi-user mode, like the team list.
- An employee hired before Canvas has no Canvas node, so its
`canvas_node_id` is null. Adding one in Dev mode fills it in.
-- The Workspace's Browser and Android tabs have no live view yet, and it
- has no timeline, replay or Take over.
+- The Workspace's Android tab has no live mirror yet. Browser now uses the
+ shared live viewer described above; timeline and replay are not available.
- With login on, an agent's Canvas writes land under the default owner:
its tool call carries no user id
([agent_workflow.py](../server/services/temporal/agent_workflow.py)).
diff --git a/server/services/employees/graph_index.py b/server/services/employees/graph_index.py
index b9d266f6..8edd89a6 100644
--- a/server/services/employees/graph_index.py
+++ b/server/services/employees/graph_index.py
@@ -16,6 +16,7 @@ from typing import Any, Dict, List, Mapping, Optional, Tuple
from constants import AI_AGENT_TYPES, WORKFLOW_TRIGGER_TYPES
from services.employees.apps import AppSpec, app_for_node_type
+from services.node_registry import get_node_class
TODO_NODE_TYPE = "writeTodos"
CANVAS_NODE_TYPE = "canvas"
@@ -34,6 +35,7 @@ class GraphIndex:
trigger_ids: Tuple[str, ...] = ()
todo_ids: Tuple[str, ...] = ()
canvas_ids: Tuple[str, ...] = ()
+ browser_ids: Tuple[str, ...] = ()
gate_ids: Tuple[str, ...] = ()
#: Apps the graph's nodes belong to, in first-seen order.
app_ids: Tuple[str, ...] = ()
@@ -64,6 +66,7 @@ def index_graph(graph: Optional[Mapping[str, Any]]) -> GraphIndex:
triggers: List[str] = []
todos: List[str] = []
canvases: List[str] = []
+ browsers: List[str] = []
gates: List[str] = []
app_ids: List[str] = []
for node in nodes:
@@ -78,6 +81,11 @@ def index_graph(graph: Optional[Mapping[str, Any]]) -> GraphIndex:
label = data.get("label") if isinstance(data, Mapping) else None
if isinstance(label, str) and label.strip():
labels[node_id] = label.strip()
+ # Browser viewing is available even when execution of the node is
+ # disabled. Discover capabilities from plugins, never type names.
+ cls = get_node_class(node_type)
+ if cls is not None and (getattr(cls, "ui_hints", {}) or {}).get("isBrowserPanel") and node_id not in browsers:
+ browsers.append(node_id)
if isinstance(data, Mapping) and data.get("disabled"):
continue
if node_type in AI_AGENT_TYPES:
@@ -100,6 +108,7 @@ def index_graph(graph: Optional[Mapping[str, Any]]) -> GraphIndex:
trigger_ids=tuple(triggers),
todo_ids=tuple(todos),
canvas_ids=tuple(canvases),
+ browser_ids=tuple(browsers),
gate_ids=tuple(gates),
app_ids=tuple(app_ids),
)
diff --git a/server/services/employees/summaries.py b/server/services/employees/summaries.py
index 50150b0d..76fe2f7e 100644
--- a/server/services/employees/summaries.py
+++ b/server/services/employees/summaries.py
@@ -223,6 +223,7 @@ async def _summary(
"control": control,
"watch_node_ids": watch,
"canvas_node_id": canvas_node_id,
+ "browser_nodes": [{"node_id": node_id, "label": graph.labels.get(node_id) or "Browser"} for node_id in graph.browser_ids],
"revision": _millis(
getattr(workflow, "updated_at", None),
getattr(employee, "updated_at", None),
diff --git a/server/tests/services/employees/test_list_employees.py b/server/tests/services/employees/test_list_employees.py
index db0da33a..393f8501 100644
--- a/server/tests/services/employees/test_list_employees.py
+++ b/server/tests/services/employees/test_list_employees.py
@@ -101,6 +101,28 @@ def test_graph_index_reads_agents_triggers_and_apps():
assert disabled.agent_ids == ()
+async def test_summary_discovers_all_browser_capabilities_including_disabled(real_database, monkeypatch):
+ from types import SimpleNamespace
+ import services.employees.graph_index as graph_index
+
+ # A third-party browser is discovered by capability; an unknown plugin
+ # called 'browser' is not trusted just because of its name.
+ monkeypatch.setattr(graph_index, "get_node_class", lambda kind: SimpleNamespace(ui_hints={"isBrowserPanel": True}) if kind == "customBrowser" else None)
+ await save(real_database, "91", "Research", graph(
+ node("first", "customBrowser", "Research browser"),
+ node("second", "customBrowser", " ", disabled=True),
+ node("not-browser", "browser"),
+ node("first", "customBrowser", "Research browser"),
+ ))
+ summary = await get_employee_summary(real_database, "91", auth_service=FakeAuth())
+ assert summary["browser_nodes"] == [
+ {"node_id": "first", "label": "Research browser"},
+ {"node_id": "second", "label": "Browser"},
+ ]
+ [listed] = await list_employee_summaries(real_database, auth_service=FakeAuth())
+ assert listed["browser_nodes"] == summary["browser_nodes"]
+
+
@pytest.mark.parametrize(
("trigger", "text"),
[