Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

Stabilize preview browser surfaces, automation, and recording (#3565)

Julius Marminge44fb34ad5383
diff --git a/apps/server/src/mcp/McpHttpServer.test.ts b/apps/server/src/mcp/McpHttpServer.test.ts
index 14bd4c205..ca3341be7 100644
--- a/apps/server/src/mcp/McpHttpServer.test.ts
+++ b/apps/server/src/mcp/McpHttpServer.test.ts
@@ -15,6 +15,7 @@ import * as PreviewAutomationBroker from "./PreviewAutomationBroker.ts";
const environmentId = EnvironmentId.make("environment-mcp-test");
const threadId = ThreadId.make("thread-mcp-test");
const tabId = PreviewTabId.make("tab-mcp-test");
+const alternateTabId = PreviewTabId.make("tab-mcp-alternate");
const invocation = {
environmentId,
threadId,
@@ -154,49 +155,53 @@ it.effect("registers annotated tools and preserves authenticated request context
Effect.gen(function* () {
const server = yield* McpServer.McpServer;
const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker;
+ const routedRequests: Array<{
+ readonly operation: string;
+ readonly tabId?: string | undefined;
+ }> = [];
const events = yield* broker.connect({
clientId: "mcp-test-client",
environmentId,
});
- yield* Stream.runForEach(events, (event) =>
- event.type === "connected"
- ? Effect.void
- : broker.respond({
- clientId: "mcp-test-client",
- connectionId: event.connectionId,
- requestId: event.request.requestId,
- ok: true,
- result:
- event.request.operation === "snapshot"
- ? {
- url: "http://example.test/",
- title: "Example",
- loading: false,
- visibleText: "Example",
- interactiveElements: [],
- accessibilityTree: {},
- consoleEntries: [],
- networkEntries: [],
- actionTimeline: [],
- screenshot: {
- mimeType: "image/png",
- data: Buffer.from("png").toString("base64"),
- width: 10,
- height: 5,
- },
- }
- : event.request.operation === "press"
- ? undefined
- : {
- available: true,
- visible: true,
- tabId,
- url: "http://example.test/",
- title: "Example",
- loading: false,
- },
- }),
- ).pipe(Effect.forkScoped);
+ yield* Stream.runForEach(events, (event) => {
+ if (event.type === "connected") return Effect.void;
+ routedRequests.push(event.request);
+ return broker.respond({
+ clientId: "mcp-test-client",
+ connectionId: event.connectionId,
+ requestId: event.request.requestId,
+ ok: true,
+ result:
+ event.request.operation === "snapshot"
+ ? {
+ url: "http://example.test/",
+ title: "Example",
+ loading: false,
+ visibleText: "Example",
+ interactiveElements: [],
+ accessibilityTree: {},
+ consoleEntries: [],
+ networkEntries: [],
+ actionTimeline: [],
+ screenshot: {
+ mimeType: "image/png",
+ data: Buffer.from("png").toString("base64"),
+ width: 10,
+ height: 5,
+ },
+ }
+ : event.request.operation === "press"
+ ? undefined
+ : {
+ available: true,
+ visible: true,
+ tabId,
+ url: "http://example.test/",
+ title: "Example",
+ loading: false,
+ },
+ });
+ }).pipe(Effect.forkScoped);
yield* Effect.yieldNow;
const statusTool = server.tools.find(({ tool }) => tool.name === "preview_status");
@@ -239,7 +244,7 @@ it.effect("registers annotated tools and preserves authenticated request context
expect(malformed.isError).toBe(true);
const snapshot = yield* server
- .callTool({ name: "preview_snapshot", arguments: {} })
+ .callTool({ name: "preview_snapshot", arguments: { tabId: alternateTabId } })
.pipe(
Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
Effect.provideService(McpSchema.McpServerClient, client),
@@ -249,6 +254,9 @@ it.effect("registers annotated tools and preserves authenticated request context
expect(snapshot.structuredContent).toMatchObject({
screenshot: { mimeType: "image/png", width: 10, height: 5 },
});
+ expect(routedRequests.find(({ operation }) => operation === "snapshot")?.tabId).toBe(
+ alternateTabId,
+ );
const press = yield* server
.callTool({ name: "preview_press", arguments: { key: "Enter" } })
diff --git a/apps/server/src/mcp/PreviewAutomationBroker.test.ts b/apps/server/src/mcp/PreviewAutomationBroker.test.ts
index 660035573..ffda427d8 100644
--- a/apps/server/src/mcp/PreviewAutomationBroker.test.ts
+++ b/apps/server/src/mcp/PreviewAutomationBroker.test.ts
@@ -15,6 +15,7 @@ import {
type PreviewAutomationStreamEvent,
} from "@t3tools/contracts";
import * as Effect from "effect/Effect";
+import * as Deferred from "effect/Deferred";
import * as Fiber from "effect/Fiber";
import * as Result from "effect/Result";
import * as Stream from "effect/Stream";
@@ -84,6 +85,187 @@ it.effect("atomically registers a connected host and correlates its response", (
),
);
+it.effect("targets multiple tabs explicitly while retaining a default tab", () =>
+ Effect.scoped(
+ Effect.gen(function* () {
+ const broker = yield* makeBroker;
+ const appTabId = PreviewTabId.make("tab-web-app");
+ const simulatorTabId = PreviewTabId.make("tab-ios-simulator");
+ const openedTabIds = [appTabId, simulatorTabId];
+ let openIndex = 0;
+ const routedRequests: RoutedRequest[] = [];
+ const requests = requestsFrom(yield* broker.connect(makeHost()));
+ yield* Stream.runForEach(requests, (request) => {
+ routedRequests.push(request);
+ return broker.respond({
+ clientId: "client-1",
+ connectionId: request.connectionId,
+ requestId: request.requestId,
+ ok: true,
+ result:
+ request.operation === "open"
+ ? { available: true, tabId: openedTabIds[openIndex++] }
+ : { url: "http://localhost:3200" },
+ });
+ }).pipe(Effect.forkScoped);
+ yield* Effect.yieldNow;
+
+ yield* broker.invoke({ scope, operation: "open", input: { reuseExistingTab: false } });
+ yield* broker.invoke({ scope, operation: "open", input: { reuseExistingTab: false } });
+ yield* broker.invoke({ scope, operation: "snapshot", input: {} });
+ yield* broker.invoke({ scope, operation: "snapshot", input: {}, tabId: appTabId });
+ yield* broker.invoke({ scope, operation: "snapshot", input: {} });
+
+ expect(routedRequests).toHaveLength(5);
+ expect(routedRequests[0]?.tabId).toBeUndefined();
+ expect(routedRequests[1]?.tabId).toBe(appTabId);
+ expect(routedRequests[2]?.tabId).toBe(simulatorTabId);
+ expect(routedRequests[2]?.tabIdExplicit).toBe(false);
+ expect(routedRequests[3]?.tabId).toBe(appTabId);
+ expect(routedRequests[3]?.tabIdExplicit).toBe(true);
+ expect(routedRequests[4]?.tabId).toBe(appTabId);
+ }),
+ ),
+);
+
+it.effect("does not let an older response replace a newer explicit tab target", () =>
+ Effect.scoped(
+ Effect.gen(function* () {
+ const broker = yield* makeBroker;
+ const olderTabId = PreviewTabId.make("tab-older-request");
+ const newerTabId = PreviewTabId.make("tab-newer-request");
+ const releaseOlderResponse = yield* Deferred.make<void>();
+ const routedRequests: RoutedRequest[] = [];
+ const requests = requestsFrom(yield* broker.connect(makeHost()));
+ yield* Stream.runForEach(requests, (request) => {
+ routedRequests.push(request);
+ const response = Effect.gen(function* () {
+ if (request.tabId === olderTabId) {
+ yield* Deferred.await(releaseOlderResponse);
+ }
+ yield* broker.respond({
+ clientId: "client-1",
+ connectionId: request.connectionId,
+ requestId: request.requestId,
+ ok: true,
+ result: { url: "http://localhost:3200" },
+ });
+ if (request.tabId === newerTabId) {
+ yield* Deferred.succeed(releaseOlderResponse, undefined);
+ }
+ });
+ return response.pipe(Effect.forkScoped, Effect.asVoid);
+ }).pipe(Effect.forkScoped);
+ yield* Effect.yieldNow;
+
+ const older = yield* broker
+ .invoke({ scope, operation: "snapshot", input: {}, tabId: olderTabId })
+ .pipe(Effect.forkScoped);
+ yield* Effect.yieldNow;
+ const newer = yield* broker
+ .invoke({ scope, operation: "snapshot", input: {}, tabId: newerTabId })
+ .pipe(Effect.forkScoped);
+ yield* Fiber.join(newer);
+ yield* Fiber.join(older);
+ yield* broker.invoke({ scope, operation: "snapshot", input: {} });
+
+ expect(routedRequests.at(-1)?.tabId).toBe(newerTabId);
+ }),
+ ),
+);
+
+it.effect("does not replace the default tab with a globally stopped recording tab", () =>
+ Effect.scoped(
+ Effect.gen(function* () {
+ const broker = yield* makeBroker;
+ const browsingTabId = PreviewTabId.make("tab-session-b");
+ const recordingTabId = PreviewTabId.make("tab-session-a-recording");
+ const routedRequests: RoutedRequest[] = [];
+ const requests = requestsFrom(yield* broker.connect(makeHost()));
+ yield* Stream.runForEach(requests, (request) => {
+ routedRequests.push(request);
+ return broker.respond({
+ clientId: "client-1",
+ connectionId: request.connectionId,
+ requestId: request.requestId,
+ ok: true,
+ result:
+ request.operation === "open"
+ ? { available: true, tabId: browsingTabId }
+ : request.operation === "recordingStop"
+ ? { id: "recording-1", tabId: recordingTabId }
+ : { url: "http://localhost:3200" },
+ });
+ }).pipe(Effect.forkScoped);
+ yield* Effect.yieldNow;
+
+ yield* broker.invoke({ scope, operation: "open", input: {} });
+ yield* broker.invoke({ scope, operation: "recordingStop", input: {} });
+ yield* broker.invoke({ scope, operation: "snapshot", input: {} });
+
+ expect(routedRequests.at(-1)?.tabId).toBe(browsingTabId);
+ }),
+ ),
+);
+
+it.effect("does not let a no-tab response suppress an earlier tab decision", () =>
+ Effect.scoped(
+ Effect.gen(function* () {
+ const broker = yield* makeBroker;
+ const initialTabId = PreviewTabId.make("tab-initial");
+ const openedTabId = PreviewTabId.make("tab-opened-late");
+ const releaseOpenResponse = yield* Deferred.make<void>();
+ const routedRequests: RoutedRequest[] = [];
+ const requests = requestsFrom(yield* broker.connect(makeHost()));
+ yield* Stream.runForEach(requests, (request) => {
+ routedRequests.push(request);
+ const marker =
+ typeof request.input === "object" && request.input !== null && "marker" in request.input
+ ? request.input.marker
+ : undefined;
+ const response = Effect.gen(function* () {
+ if (marker === "older") {
+ yield* Deferred.await(releaseOpenResponse);
+ }
+ yield* broker.respond({
+ clientId: "client-1",
+ connectionId: request.connectionId,
+ requestId: request.requestId,
+ ok: true,
+ result:
+ request.operation === "open"
+ ? { available: true, tabId: marker === "older" ? openedTabId : initialTabId }
+ : { url: "http://localhost:3200" },
+ });
+ if (marker === "newer") {
+ yield* Deferred.succeed(releaseOpenResponse, undefined);
+ }
+ });
+ return response.pipe(Effect.forkScoped, Effect.asVoid);
+ }).pipe(Effect.forkScoped);
+ yield* Effect.yieldNow;
+
+ yield* broker.invoke({ scope, operation: "open", input: {} });
+ const older = yield* broker
+ .invoke({
+ scope,
+ operation: "open",
+ input: { marker: "older", reuseExistingTab: false },
+ })
+ .pipe(Effect.forkScoped);
+ yield* Effect.yieldNow;
+ const newer = yield* broker
+ .invoke({ scope, operation: "snapshot", input: { marker: "newer" } })
+ .pipe(Effect.forkScoped);
+ yield* Fiber.join(newer);
+ yield* Fiber.join(older);
+ yield* broker.invoke({ scope, operation: "snapshot", input: {} });
+
+ expect(routedRequests.at(-1)?.tabId).toBe(openedTabId);
+ }),
+ ),
+);
+
it.effect("announces a live replacement stream before delivering requests", () =>
Effect.scoped(
Effect.gen(function* () {
@@ -632,7 +814,9 @@ it.effect("fails over a pinned provider session only after its host disconnects"
Effect.scoped(
Effect.gen(function* () {
const broker = yield* makeBroker;
+ const firstTabId = PreviewTabId.make("tab-on-first-host");
let firstConnectionId = "";
+ let secondRoutedTabId: PreviewTabId | undefined;
const firstRequests = requestsFrom(
yield* broker.connect(makeHost({ clientId: "client-first" })),
(connectionId) => {
@@ -648,18 +832,19 @@ it.effect("fails over a pinned provider session only after its host disconnects"
connectionId: request.connectionId,
requestId: request.requestId,
ok: true,
- result: "first",
+ result: request.operation === "open" ? { host: "first", tabId: firstTabId } : "first",
}),
).pipe(Effect.forkScoped);
- yield* Stream.runForEach(secondRequests, (request) =>
- broker.respond({
+ yield* Stream.runForEach(secondRequests, (request) => {
+ secondRoutedTabId = request.tabId;
+ return broker.respond({
clientId: "client-second",
connectionId: request.connectionId,
requestId: request.requestId,
ok: true,
result: "second",
- }),
- ).pipe(Effect.forkScoped);
+ });
+ }).pipe(Effect.forkScoped);
yield* Effect.yieldNow;
yield* broker.focusHost({
@@ -668,7 +853,10 @@ it.effect("fails over a pinned provider session only after its host disconnects"
connectionId: firstConnectionId,
focused: true,
});
- expect(yield* broker.invoke<string>({ scope, operation: "status", input: {} })).toBe("first");
+ expect(yield* broker.invoke({ scope, operation: "open", input: {} })).toEqual({
+ host: "first",
+ tabId: firstTabId,
+ });
yield* Fiber.interrupt(firstConsumer);
yield* Effect.yieldNow;
@@ -676,6 +864,7 @@ it.effect("fails over a pinned provider session only after its host disconnects"
expect(yield* broker.invoke<string>({ scope, operation: "status", input: {} })).toBe(
"second",
);
+ expect(secondRoutedTabId).toBeUndefined();
}),
),
);
@@ -740,6 +929,53 @@ it.effect("keeps a replacement stream authoritative when the old stream finalize
),
);
+it.effect("does not carry a tab id across a replacement automation stream", () =>
+ Effect.scoped(
+ Effect.gen(function* () {
+ const broker = yield* makeBroker;
+ const openedTabId = PreviewTabId.make("tab-first-webcontents");
+ const firstRequests = requestsFrom(yield* broker.connect(makeHost()));
+ yield* Stream.runForEach(firstRequests, (request) =>
+ broker.respond({
+ clientId: "client-1",
+ connectionId: request.connectionId,
+ requestId: request.requestId,
+ ok: true,
+ result:
+ request.operation === "open"
+ ? { host: "first", tabId: openedTabId }
+ : { host: "first" },
+ }),
+ ).pipe(Effect.forkScoped);
+ yield* Effect.yieldNow;
+
+ expect(yield* broker.invoke({ scope, operation: "open", input: {} })).toEqual({
+ host: "first",
+ tabId: openedTabId,
+ });
+
+ const routedRequests: RoutedRequest[] = [];
+ const replacementRequests = requestsFrom(yield* broker.connect(makeHost()));
+ yield* Stream.runForEach(replacementRequests, (request) => {
+ routedRequests.push(request);
+ return broker.respond({
+ clientId: "client-1",
+ connectionId: request.connectionId,
+ requestId: request.requestId,
+ ok: true,
+ result: "replacement",
+ });
+ }).pipe(Effect.forkScoped);
+ yield* Effect.yieldNow;
+
+ expect(yield* broker.invoke<string>({ scope, operation: "status", input: {} })).toBe(
+ "replacement",
+ );
+ expect(routedRequests.at(-1)?.tabId).toBeUndefined();
+ }),
+ ),
+);
+
it.effect("fails requests assigned to the stream that is replaced", () =>
Effect.scoped(
Effect.gen(function* () {
diff --git a/apps/server/src/mcp/PreviewAutomationBroker.ts b/apps/server/src/mcp/PreviewAutomationBroker.ts
index 688383d7b..7ed77aabd 100644
--- a/apps/server/src/mcp/PreviewAutomationBroker.ts
+++ b/apps/server/src/mcp/PreviewAutomationBroker.ts
@@ -13,13 +13,13 @@ import {
PreviewAutomationTargetNotEditableError,
PreviewAutomationTimeoutError,
PreviewAutomationUnsupportedClientError,
+ PreviewTabId,
type PreviewAutomationError,
type PreviewAutomationOperation,
type PreviewAutomationHost,
type PreviewAutomationHostFocus,
type PreviewAutomationResponse,
type PreviewAutomationStreamEvent,
- type PreviewTabId,
} from "@t3tools/contracts";
import * as Context from "effect/Context";
import * as Clock from "effect/Clock";
@@ -29,6 +29,7 @@ import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
import * as Option from "effect/Option";
import * as Queue from "effect/Queue";
+import * as Schema from "effect/Schema";
import * as Stream from "effect/Stream";
import * as SynchronizedRef from "effect/SynchronizedRef";
@@ -79,6 +80,8 @@ interface HostAssignment {
readonly connectionId: ClientConnection["connectionId"];
readonly queue: ClientConnection["queue"];
readonly expiresAt: number;
+ readonly tabId?: PreviewTabId;
+ readonly tabSequence?: number;
}
interface PreviewAutomationRequestErrorContext {
@@ -144,6 +147,14 @@ const selectorDiagnosticsFromInput = (
const hostAssignmentKey = (scope: McpInvocationContext.McpInvocationScope): string =>
`${scope.environmentId}\u0000${scope.providerSessionId}`;
+const isPreviewTabId = Schema.is(PreviewTabId);
+
+const readResultTabId = (result: unknown): PreviewTabId | null | undefined => {
+ if (typeof result !== "object" || result === null || !("tabId" in result)) return undefined;
+ const tabId = result.tabId;
+ return tabId === null || isPreviewTabId(tabId) ? tabId : undefined;
+};
+
const supportsOperation = (
connection: ClientConnection,
operation: PreviewAutomationOperation,
@@ -454,15 +465,24 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() {
if (!hasLiveAssignment) assignments.delete(assignmentKey);
return [undefined, { ...current, assignments }] as const;
}
+ const canReuseAssignedTab =
+ assigned !== undefined &&
+ assigned.connectionId === connection.connectionId &&
+ assigned.queue === connection.queue;
assignments.set(assignmentKey, {
clientId: connection.clientId,
connectionId: connection.connectionId,
queue: connection.queue,
expiresAt: input.scope.expiresAt,
+ ...(canReuseAssignedTab && assigned.tabId !== undefined ? { tabId: assigned.tabId } : {}),
+ ...(canReuseAssignedTab && assigned.tabSequence !== undefined
+ ? { tabSequence: assigned.tabSequence }
+ : {}),
});
- const requestId = `preview-${current.requestSequence}`;
- const tabId = input.tabId;
+ const requestSequence = current.requestSequence;
+ const requestId = `preview-${requestSequence}`;
+ const tabId = input.tabId ?? (canReuseAssignedTab ? assigned.tabId : undefined);
const selectorDiagnostics = selectorDiagnosticsFromInput(input.input);
const context: PreviewAutomationRequestErrorContext = {
operation: input.operation,
@@ -480,7 +500,7 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() {
const pending = new Map(current.pending);
pending.set(requestId, { queue: connection.queue, deferred, context });
return [
- { connection, requestId, requestContext: context },
+ { connection, requestId, requestContext: context, requestSequence },
{ ...current, assignments, pending, requestSequence: current.requestSequence + 1 },
] as const;
});
@@ -493,7 +513,7 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() {
providerInstanceId: input.scope.providerInstanceId,
});
}
- const { connection, requestId, requestContext } = route;
+ const { connection, requestId, requestContext, requestSequence } = route;
const removePending = SynchronizedRef.update(state, (next) => {
if (!next.pending.has(requestId)) return next;
const pending = new Map(next.pending);
@@ -508,6 +528,7 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() {
requestId,
threadId: input.scope.threadId,
tabId: requestContext.tabId,
+ tabIdExplicit: input.tabId !== undefined,
operation: input.operation,
input: input.input,
timeoutMs,
@@ -526,7 +547,36 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() {
onSome: (value) => Effect.succeed(value as A),
});
});
- return yield* awaitResponse().pipe(Effect.ensuring(removePending));
+ const result = yield* awaitResponse().pipe(Effect.ensuring(removePending));
+ // A stop artifact identifies the globally recorded tab, not the caller's browsing target.
+ const responseTabId = input.operation === "recordingStop" ? undefined : readResultTabId(result);
+ const resultTabId = responseTabId === undefined ? input.tabId : responseTabId;
+ if (resultTabId === undefined) return result;
+ const assignmentKey = hostAssignmentKey(input.scope);
+ yield* SynchronizedRef.update(state, (current) => {
+ const assignment = current.assignments.get(assignmentKey);
+ if (
+ !assignment ||
+ assignment.connectionId !== connection.connectionId ||
+ assignment.queue !== connection.queue ||
+ (assignment.tabSequence ?? -1) > requestSequence
+ ) {
+ return current;
+ }
+ const assignments = new Map(current.assignments);
+ if (resultTabId === null) {
+ const { tabId: _tabId, ...withoutTabId } = assignment;
+ assignments.set(assignmentKey, { ...withoutTabId, tabSequence: requestSequence });
+ } else {
+ assignments.set(assignmentKey, {
+ ...assignment,
+ ...(resultTabId === undefined ? {} : { tabId: resultTabId }),
+ tabSequence: requestSequence,
+ });
+ }
+ return { ...current, assignments };
+ });
+ return result;
});
return PreviewAutomationBroker.of({ connect, focusHost, respond, invoke });
diff --git a/apps/server/src/mcp/toolkits/preview/handlers.ts b/apps/server/src/mcp/toolkits/preview/handlers.ts
index 700b81a0b..64d6ba02b 100644
--- a/apps/server/src/mcp/toolkits/preview/handlers.ts
+++ b/apps/server/src/mcp/toolkits/preview/handlers.ts
@@ -6,6 +6,7 @@ import type {
PreviewAutomationResizeResult,
PreviewAutomationSnapshot,
PreviewAutomationStatus,
+ PreviewTabId,
} from "@t3tools/contracts";
import * as McpInvocationContext from "../../McpInvocationContext.ts";
@@ -16,6 +17,7 @@ const invoke = Effect.fn("PreviewToolkit.invoke")(function* <A>(
operation: PreviewAutomationOperation,
input: unknown,
timeoutMs?: number,
+ tabId?: PreviewTabId,
): Effect.fn.Return<
A,
import("@t3tools/contracts").PreviewAutomationError,
@@ -28,31 +30,49 @@ const invoke = Effect.fn("PreviewToolkit.invoke")(function* <A>(
operation,
input,
...(timeoutMs === undefined ? {} : { timeoutMs }),
+ ...(tabId === undefined ? {} : { tabId }),
});
});
+const invokeTargeted = <A>(
+ operation: PreviewAutomationOperation,
+ input: {
+ readonly tabId?: PreviewTabId | undefined;
+ readonly [key: string]: unknown;
+ },
+ timeoutMs?: number,
+) => {
+ const { tabId, ...operationInput } = input;
+ return invoke<A>(operation, operationInput, timeoutMs, tabId);
+};
+
const handlers = {
- preview_status: () => invoke<PreviewAutomationStatus>("status", {}),
+ preview_status: (input) => invokeTargeted<PreviewAutomationStatus>("status", input ?? {}),
preview_open: (input) =>
- invoke<PreviewAutomationStatus>("open", {
+ invokeTargeted<PreviewAutomationStatus>("open", {
...input,
show: input.show ?? true,
reuseExistingTab: input.reuseExistingTab ?? true,
}),
- preview_navigate: (input) => invoke<PreviewAutomationStatus>("navigate", input, input.timeoutMs),
+ preview_navigate: (input) =>
+ invokeTargeted<PreviewAutomationStatus>("navigate", input, input.timeoutMs),
preview_resize: (input) =>
- invoke<PreviewAutomationResizeResult>("resize", input, input.timeoutMs),
- preview_snapshot: () => invoke<PreviewAutomationSnapshot>("snapshot", {}),
- preview_click: (input) => invoke<void>("click", input, input.timeoutMs).pipe(Effect.as(null)),
- preview_type: (input) => invoke<void>("type", input, input.timeoutMs).pipe(Effect.as(null)),
- preview_press: (input) => invoke<void>("press", input).pipe(Effect.as(null)),
- preview_scroll: (input) => invoke<void>("scroll", input).pipe(Effect.as(null)),
+ invokeTargeted<PreviewAutomationResizeResult>("resize", input, input.timeoutMs),
+ preview_snapshot: (input) => invokeTargeted<PreviewAutomationSnapshot>("snapshot", input ?? {}),
+ preview_click: (input) =>
+ invokeTargeted<void>("click", input, input.timeoutMs).pipe(Effect.as(null)),
+ preview_type: (input) =>
+ invokeTargeted<void>("type", input, input.timeoutMs).pipe(Effect.as(null)),
+ preview_press: (input) => invokeTargeted<void>("press", input).pipe(Effect.as(null)),
+ preview_scroll: (input) => invokeTargeted<void>("scroll", input).pipe(Effect.as(null)),
preview_evaluate: (input) =>
- invoke<unknown>("evaluate", input).pipe(Effect.map((result) => result ?? null)),
+ invokeTargeted<unknown>("evaluate", input).pipe(Effect.map((result) => result ?? null)),
preview_wait_for: (input) =>
- invoke<void>("waitFor", input, input.timeoutMs).pipe(Effect.as(null)),
- preview_recording_start: () => invoke<PreviewAutomationRecordingStatus>("recordingStart", {}),
- preview_recording_stop: () => invoke<PreviewAutomationRecordingArtifact>("recordingStop", {}),
+ invokeTargeted<void>("waitFor", input, input.timeoutMs).pipe(Effect.as(null)),
+ preview_recording_start: (input) =>
+ invokeTargeted<PreviewAutomationRecordingStatus>("recordingStart", input ?? {}),
+ preview_recording_stop: (input) =>
+ invokeTargeted<PreviewAutomationRecordingArtifact>("recordingStop", input ?? {}),
} satisfies Parameters<typeof PreviewToolkit.toLayer>[0];
const { preview_snapshot, ...standardHandlers } = handlers;
diff --git a/apps/server/src/mcp/toolkits/preview/tools.test.ts b/apps/server/src/mcp/toolkits/preview/tools.test.ts
index 1347e0db0..d00ff459b 100644
--- a/apps/server/src/mcp/toolkits/preview/tools.test.ts
+++ b/apps/server/src/mcp/toolkits/preview/tools.test.ts
@@ -27,6 +27,10 @@ it("exports provider-compatible object schemas with described parameters", () =>
expect(schema.type, `${tool.name} must export a top-level object schema`).toBe("object");
expect(schema.anyOf, `${tool.name} must not export a root anyOf`).toBeUndefined();
expect(schema.oneOf, `${tool.name} must not export a root oneOf`).toBeUndefined();
+ expect(
+ schema.properties?.tabId,
+ `${tool.name} must allow an explicit collaborative browser tab target`,
+ ).toBeDefined();
for (const [field, fieldSchema] of Object.entries(schema.properties ?? {})) {
expect(
schemaHasDescription(fieldSchema),
diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts
index d9567b01c..c729fc20e 100644
--- a/apps/server/src/mcp/toolkits/preview/tools.ts
+++ b/apps/server/src/mcp/toolkits/preview/tools.ts
@@ -12,6 +12,7 @@ import {
PreviewAutomationScrollInput,
PreviewAutomationSnapshot,
PreviewAutomationStatus,
+ PreviewAutomationTabTargetInput,
PreviewAutomationTypeInput,
PreviewAutomationWaitForInput,
} from "@t3tools/contracts";
@@ -37,7 +38,8 @@ const readonlyBrowserTool = <T extends Tool.Any>(tool: T): T =>
export const PreviewStatusTool = Tool.make("preview_status", {
description:
- "Report whether the scoped thread has an automation-capable desktop preview, including its active tab, URL, title, visibility, loading state, viewport mode, and measured CSS-pixel size.",
+ "Report whether a collaborative browser tab is automation-capable, including its URL, title, visibility, loading state, viewport mode, and measured CSS-pixel size. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab.",
+ parameters: PreviewAutomationTabTargetInput,
success: PreviewAutomationStatus,
failure: PreviewAutomationError,
dependencies,
@@ -50,7 +52,7 @@ export const PreviewStatusTool = Tool.make("preview_status", {
export const PreviewOpenTool = browserTool(
Tool.make("preview_open", {
description:
- "Show and initialize the browser preview for the scoped thread, optionally reusing its current tab and navigating to a URL.",
+ "Show and initialize a collaborative browser tab. Pass tabId to reuse a specific existing tab, set reuseExistingTab=false to create another tab, or omit both to use this agent session's current tab.",
parameters: PreviewAutomationOpenInput,
success: PreviewAutomationStatus,
failure: PreviewAutomationError,
@@ -63,7 +65,7 @@ export const PreviewOpenTool = browserTool(
export const PreviewNavigateTool = safeBrowserTool(
Tool.make("preview_navigate", {
description:
- "Navigate the active collaborative browser tab. Pass {url:'https://t3.chat'} for a website, or {target:{kind:'environment-port',port:5173}} for a dev server in the current environment. Exactly one of url or target is required. Defaults to waiting for page loading to stop.",
+ "Navigate a collaborative browser tab. Pass tabId to target a specific tab, plus {url:'https://t3.chat'} for a website or {target:{kind:'environment-port',port:5173}} for a dev server. Exactly one of url or target is required.",
parameters: PreviewAutomationNavigateInput,
success: PreviewAutomationStatus,
failure: PreviewAutomationError,
@@ -74,7 +76,7 @@ export const PreviewNavigateTool = safeBrowserTool(
export const PreviewResizeTool = safeBrowserTool(
Tool.make("preview_resize", {
description:
- "Set the active collaborative browser tab to fill-panel sizing, an independently resizable freeform size, or a Chrome-standard device preset. Use {mode:'fill'}, {mode:'freeform',width:1024,height:768}, or {mode:'preset',preset:'iphone-12-pro',orientation:'portrait'}. This changes CSS layout breakpoints without changing the desktop browser user agent.",
+ "Resize a collaborative browser tab, optionally selected by tabId. Use {mode:'fill'}, {mode:'freeform',width:1024,height:768}, or {mode:'preset',preset:'iphone-12-pro',orientation:'portrait'}. This changes CSS layout breakpoints without changing the desktop browser user agent.",
parameters: PreviewAutomationResizeInput,
success: PreviewAutomationResizeResult,
failure: PreviewAutomationError,
@@ -87,7 +89,8 @@ export const PreviewResizeTool = safeBrowserTool(
export const PreviewSnapshotTool = readonlyBrowserTool(
Tool.make("preview_snapshot", {
description:
- "Inspect the current page before interacting. Returns URL/title/loading state, visible text, semantic interactive elements with reusable selectors and coordinates, accessibility data, recent console/network failures, action history, and a PNG screenshot.",
+ "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot.",
+ parameters: PreviewAutomationTabTargetInput,
success: PreviewAutomationSnapshot,
failure: PreviewAutomationError,
dependencies,
@@ -97,7 +100,7 @@ export const PreviewSnapshotTool = readonlyBrowserTool(
export const PreviewClickTool = browserTool(
Tool.make("preview_click", {
description:
- "Click exactly one page target. Prefer locator with a Playwright selector such as role=button[name='Send']; selector accepts legacy CSS; x and y are viewport CSS pixels and must be supplied together. Call preview_snapshot first when the target is unknown.",
+ "Click exactly one target in the tab selected by tabId, or this agent session's current tab when omitted. Prefer a Playwright locator; selector accepts legacy CSS; x and y must be supplied together.",
parameters: PreviewAutomationClickInput,
success: Schema.Null,
failure: PreviewAutomationError,
@@ -108,7 +111,7 @@ export const PreviewClickTool = browserTool(
export const PreviewTypeTool = browserTool(
Tool.make("preview_type", {
description:
- "Insert literal text into one input. Prefer locator with a Playwright role/text selector; selector accepts legacy CSS. If neither is supplied, types into the currently focused element. Set clear=true to replace existing text.",
+ "Insert literal text into one input in the tab selected by tabId, or this agent session's current tab when omitted. Prefer a Playwright locator; set clear=true to replace existing text.",
parameters: PreviewAutomationTypeInput,
success: Schema.Null,
failure: PreviewAutomationError,
@@ -119,7 +122,7 @@ export const PreviewTypeTool = browserTool(
export const PreviewPressTool = browserTool(
Tool.make("preview_press", {
description:
- "Press one keyboard key in the active page, for example {key:'Enter'}, {key:'Escape'}, or {key:'a',modifiers:['Meta']}. This targets the page's current focus.",
+ "Press one keyboard key in the tab selected by tabId, or this agent session's current tab when omitted. Examples: {key:'Enter'}, {key:'Escape'}, or {key:'a',modifiers:['Meta']}.",
parameters: PreviewAutomationPressInput,
success: Schema.Null,
failure: PreviewAutomationError,
@@ -130,7 +133,7 @@ export const PreviewPressTool = browserTool(
export const PreviewScrollTool = safeBrowserTool(
Tool.make("preview_scroll", {
description:
- "Scroll by CSS pixels. Positive deltaY scrolls down and positive deltaX scrolls right. Without locator/selector it scrolls the viewport; otherwise it scrolls that container. At least one delta is required.",
+ "Scroll the tab selected by tabId, or this agent session's current tab when omitted. Positive deltaY scrolls down and positive deltaX scrolls right; a locator/selector targets a container.",
parameters: PreviewAutomationScrollInput,
success: Schema.Null,
failure: PreviewAutomationError,
@@ -141,7 +144,7 @@ export const PreviewScrollTool = safeBrowserTool(
export const PreviewEvaluateTool = browserTool(
Tool.make("preview_evaluate", {
description:
- "Evaluate a JavaScript expression in the page's main frame and return a serializable result up to 64 KB. Prefer preview_snapshot and semantic click/type/wait tools; use this for inspection or interactions those tools cannot express. The expression may mutate page state.",
+ "Evaluate JavaScript in the tab selected by tabId, or this agent session's current tab when omitted. Returns a serializable result up to 64 KB; the expression may mutate page state.",
parameters: PreviewAutomationEvaluateInput,
success: Schema.Unknown,
failure: PreviewAutomationError,
@@ -152,7 +155,7 @@ export const PreviewEvaluateTool = browserTool(
export const PreviewWaitForTool = readonlyBrowserTool(
Tool.make("preview_wait_for", {
description:
- "Wait until all supplied conditions match: a Playwright locator, legacy CSS selector, visible-text substring, and/or URL substring. Provide at least one condition. Defaults to 15 seconds, maximum 60 seconds.",
+ "Wait in the tab selected by tabId, or this agent session's current tab when omitted, until all supplied locator, selector, text, and URL conditions match.",
parameters: PreviewAutomationWaitForInput,
success: Schema.Null,
failure: PreviewAutomationError,
@@ -163,7 +166,8 @@ export const PreviewWaitForTool = readonlyBrowserTool(
export const PreviewRecordingStartTool = safeBrowserTool(
Tool.make("preview_recording_start", {
description:
- "Start recording the active collaborative browser tab while keeping it interactive for both agent and human use.",
+ "Start recording the collaborative browser tab selected by tabId, or this agent session's current tab when omitted.",
+ parameters: PreviewAutomationTabTargetInput,
success: PreviewAutomationRecordingStatus,
failure: PreviewAutomationError,
dependencies,
@@ -173,6 +177,7 @@ export const PreviewRecordingStartTool = safeBrowserTool(
export const PreviewRecordingStopTool = safeBrowserTool(
Tool.make("preview_recording_stop", {
description: "Stop the active browser recording and save it as a local evidence artifact.",
+ parameters: PreviewAutomationTabTargetInput,
success: PreviewAutomationRecordingArtifact,
failure: PreviewAutomationError,
dependencies,
diff --git a/apps/server/src/provider/Layers/CursorAdapter.test.ts b/apps/server/src/provider/Layers/CursorAdapter.test.ts
index 73dc09676..89e9c56eb 100644
--- a/apps/server/src/provider/Layers/CursorAdapter.test.ts
+++ b/apps/server/src/provider/Layers/CursorAdapter.test.ts
@@ -1021,24 +1021,19 @@ cursorAdapterTestLayer("CursorAdapterLive", (it) => {
assert.equal(turnCompleted.payload.stopReason, "cancelled");
}
- const requests = yield* waitForJsonLogMatch(
- requestLogPath,
- (entry) => entry.method === "session/cancel",
- );
+ const isCancelledApprovalResponse = (entry: Record<string, unknown>) =>
+ !("method" in entry) &&
+ typeof entry.result === "object" &&
+ entry.result !== null &&
+ "outcome" in entry.result &&
+ typeof entry.result.outcome === "object" &&
+ entry.result.outcome !== null &&
+ "outcome" in entry.result.outcome &&
+ entry.result.outcome.outcome === "cancelled";
+ yield* waitForJsonLogMatch(requestLogPath, (entry) => entry.method === "session/cancel");
+ const requests = yield* waitForJsonLogMatch(requestLogPath, isCancelledApprovalResponse);
assert.isTrue(requests.some((entry) => entry.method === "session/cancel"));
- assert.isTrue(
- requests.some(
- (entry) =>
- !("method" in entry) &&
- typeof entry.result === "object" &&
- entry.result !== null &&
- "outcome" in entry.result &&
- typeof entry.result.outcome === "object" &&
- entry.result.outcome !== null &&
- "outcome" in entry.result.outcome &&
- entry.result.outcome.outcome === "cancelled",
- ),
- );
+ assert.isTrue(requests.some(isCancelledApprovalResponse));
yield* adapter.stopSession(threadId);
}),
diff --git a/apps/web/src/browser/BrowserSurfaceSlot.tsx b/apps/web/src/browser/BrowserSurfaceSlot.tsx
index 90769f8fb..de74cfa2a 100644
--- a/apps/web/src/browser/BrowserSurfaceSlot.tsx
+++ b/apps/web/src/browser/BrowserSurfaceSlot.tsx
@@ -1,8 +1,8 @@
"use client";
-import { useEffect, useRef } from "react";
+import { useLayoutEffect, useRef } from "react";
-import { useBrowserSurfaceStore } from "./browserSurfaceStore";
+import { acquireBrowserSurface } from "./browserSurfaceStore";
export function BrowserSurfaceSlot(props: {
readonly tabId: string;
@@ -12,13 +12,13 @@ export function BrowserSurfaceSlot(props: {
const { tabId, visible, className } = props;
const elementRef = useRef<HTMLDivElement | null>(null);
- useEffect(() => {
+ useLayoutEffect(() => {
const element = elementRef.current;
if (!element) return;
+ const lease = acquireBrowserSurface(tabId);
const update = () => {
const rect = element.getBoundingClientRect();
- useBrowserSurfaceStore.getState().present(
- tabId,
+ lease.present(
{
x: Math.round(rect.x),
y: Math.round(rect.y),
@@ -37,7 +37,7 @@ export function BrowserSurfaceSlot(props: {
observer.disconnect();
window.removeEventListener("resize", update);
window.removeEventListener("scroll", update, true);
- useBrowserSurfaceStore.getState().hide(tabId);
+ lease.release();
};
}, [tabId, visible]);
diff --git a/apps/web/src/browser/HostedBrowserWebview.tsx b/apps/web/src/browser/HostedBrowserWebview.tsx
index 2907a067a..a72da49c6 100644
--- a/apps/web/src/browser/HostedBrowserWebview.tsx
+++ b/apps/web/src/browser/HostedBrowserWebview.tsx
@@ -8,13 +8,13 @@ import { previewBridge } from "~/components/preview/previewBridge";
import { usePreviewBridge } from "~/components/preview/usePreviewBridge";
import { cn } from "~/lib/utils";
-import { useActiveBrowserRecordingTabId } from "./browserRecording";
+import { stopBrowserRecording, useActiveBrowserRecordingTabId } from "./browserRecording";
import { resolveBrowserSurfacePanelRect, useBrowserSurfaceStore } from "./browserSurfaceStore";
import { browserViewportSettingKey } from "./browserViewportLayout";
-import { reconcileLockedAspectRatio } from "./browserDeviceToolbarState";
import { BrowserDeviceToolbar } from "./BrowserDeviceToolbar";
import { BrowserViewportResizeHandles } from "./BrowserViewportResizeHandles";
import { acquireDesktopTab, type AcquiredDesktopTab } from "./desktopTabLifetime";
+import { resolveHostedBrowserWebviewWrapperStyle } from "./hostedBrowserWebviewStyle";
import { usePreviewWebviewConfig } from "./previewWebviewConfigState";
import { useBrowserViewportResize } from "./useBrowserViewportResize";
@@ -46,7 +46,8 @@ export function HostedBrowserWebview(props: {
const tabLeaseRef = useRef<AcquiredDesktopTab | null>(null);
const wrapperRef = useRef<HTMLDivElement | null>(null);
const webviewRef = useRef<ElectronWebview | null>(null);
- const [lockedAspectRatio, setLockedAspectRatio] = useState<number | null>(null);
+ const [aspectRatioLocked, setAspectRatioLocked] = useState(false);
+ const activeRecordingTabId = useActiveBrowserRecordingTabId();
const presentation = useBrowserSurfaceStore(
useShallow((state) => {
const current = state.byTabId[tabId];
@@ -56,10 +57,13 @@ export function HostedBrowserWebview(props: {
};
}),
);
- const recording = useActiveBrowserRecordingTabId() === tabId;
-
usePreviewBridge({ threadRef, tabId });
+ useEffect(() => {
+ if (presentation.visible || activeRecordingTabId !== tabId) return;
+ void stopBrowserRecording(tabId).catch(() => undefined);
+ }, [activeRecordingTabId, presentation.visible, tabId]);
+
useEffect(() => {
const lease = acquireDesktopTab(tabId);
tabLeaseRef.current = lease;
@@ -115,9 +119,11 @@ export function HostedBrowserWebview(props: {
const viewportHeight = viewport._tag === "fill" ? null : viewport.height;
const viewportAspectRatio =
viewportWidth === null || viewportHeight === null ? null : viewportWidth / viewportHeight;
- useEffect(() => {
- setLockedAspectRatio((current) => reconcileLockedAspectRatio(current, viewportAspectRatio));
- }, [viewportAspectRatio]);
+ const lockedAspectRatio =
+ aspectRatioLocked && viewportAspectRatio !== null ? viewportAspectRatio : null;
+ const handleAspectRatioChange = useCallback((aspectRatio: number | null) => {
+ setAspectRatioLocked(aspectRatio !== null);
+ }, []);
const hiddenSize =
viewport._tag !== "fill"
? {
@@ -170,24 +176,11 @@ export function HostedBrowserWebview(props: {
if (!config) return null;
- const wrapperStyle =
- active && lastRect
- ? {
- left: lastRect.x,
- top: lastRect.y,
- width: lastRect.width,
- height: lastRect.height,
- zIndex: 30,
- pointerEvents: "auto" as const,
- }
- : {
- left: 0,
- top: 0,
- width: hiddenSize.width,
- height: hiddenSize.height,
- zIndex: recording ? 0 : -1,
- pointerEvents: "none" as const,
- };
+ const wrapperStyle = resolveHostedBrowserWebviewWrapperStyle({
+ active,
+ rect: lastRect,
+ hiddenSize,
+ });
return (
<div
@@ -203,7 +196,7 @@ export function HostedBrowserWebview(props: {
setting={effectiveViewport}
width={Math.max(1, Math.round(containerSize.width))}
aspectRatio={lockedAspectRatio}
- onAspectRatioChange={setLockedAspectRatio}
+ onAspectRatioChange={handleAspectRatioChange}
onChange={commitViewportChange}
/>
) : null}
diff --git a/apps/web/src/browser/browserRecording.test.ts b/apps/web/src/browser/browserRecording.test.ts
new file mode 100644
index 000000000..9810987c9
--- /dev/null
+++ b/apps/web/src/browser/browserRecording.test.ts
@@ -0,0 +1,294 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test";
+
+const { events, onFrame, registrySet, save, startScreencast, stopScreencast, surfaceState } =
+ vi.hoisted(() => {
+ const events: string[] = [];
+ const surfaceState = {
+ byTabId: {} as Record<string, unknown>,
+ };
+ return {
+ events,
+ onFrame: vi.fn(() => vi.fn()),
+ registrySet: vi.fn((_atom: unknown, value: string | null) => {
+ events.push(value === null ? "clear" : `publish:${value}`);
+ }),
+ save: vi.fn(async () => ({
+ id: "recording-test",
+ tabId: "recording-tab",
+ path: "/tmp/recording-test.webm",
+ mimeType: "video/webm" as const,
+ sizeBytes: 0,
+ createdAt: "2026-06-26T00:00:00.000Z",
+ })),
+ startScreencast: vi.fn(async () => {
+ events.push("start-screencast");
+ }),
+ stopScreencast: vi.fn(async () => undefined),
+ surfaceState,
+ };
+ });
+
+vi.mock("~/components/preview/previewBridge", () => ({
+ previewBridge: {
+ recording: { onFrame, save, startScreencast, stopScreencast },
+ },
+}));
+
+vi.mock("~/rpc/atomRegistry", () => ({
+ appAtomRegistry: { set: registrySet },
+}));
+
+vi.mock("./browserSurfaceStore", () => ({
+ useBrowserSurfaceStore: {
+ getState: () => surfaceState,
+ },
+}));
+
+import {
+ BROWSER_RECORDING_STARTUP_SETTLE_TIMEOUT_MS,
+ BrowserRecordingConflictError,
+ BrowserRecordingOperationError,
+ BrowserRecordingRequiresVisibleTabError,
+ startBrowserRecording,
+ stopBrowserRecording,
+} from "./browserRecording";
+
+class FakeMediaRecorder {
+ static isTypeSupported(): boolean {
+ return true;
+ }
+
+ state: RecordingState = "inactive";
+ private readonly listeners = new Map<string, Set<EventListenerOrEventListenerObject>>();
+
+ addEventListener(type: string, listener: EventListenerOrEventListenerObject): void {
+ const listeners = this.listeners.get(type) ?? new Set();
+ listeners.add(listener);
+ this.listeners.set(type, listeners);
+ }
+
+ start(): void {
+ this.state = "recording";
+ }
+
+ stop(): void {
+ this.state = "inactive";
+ for (const listener of this.listeners.get("stop") ?? []) {
+ if (typeof listener === "function") listener(new Event("stop"));
+ else listener.handleEvent(new Event("stop"));
+ }
+ }
+}
+
+describe("browser recording", () => {
+ beforeEach(() => {
+ events.length = 0;
+ surfaceState.byTabId = {
+ "recording-tab": {
+ visible: true,
+ rect: { x: 0, y: 0, width: 800, height: 600 },
+ content: { x: 0, y: 0, width: 800, height: 600, scale: 1, scrollLeft: 0, scrollTop: 0 },
+ },
+ };
+ vi.clearAllMocks();
+ vi.stubGlobal("window", globalThis);
+ vi.stubGlobal("MediaRecorder", FakeMediaRecorder as unknown as typeof MediaRecorder);
+ vi.stubGlobal("document", {
+ createElement: () => ({
+ width: 0,
+ height: 0,
+ captureStream: () => ({}),
+ getContext: () => ({ drawImage: vi.fn() }),
+ }),
+ });
+ });
+
+ afterEach(() => {
+ vi.useRealTimers();
+ vi.unstubAllGlobals();
+ });
+
+ it("starts recording for a visible tab", async () => {
+ await startBrowserRecording("recording-tab");
+
+ expect(events).toEqual(["start-screencast", "publish:recording-tab"]);
+
+ await stopBrowserRecording("recording-tab");
+ });
+
+ it("rejects recording for a hidden tab before starting screencast", async () => {
+ surfaceState.byTabId = {
+ "recording-tab": {
+ visible: false,
+ rect: { x: 0, y: 0, width: 800, height: 600 },
+ content: { x: 0, y: 0, width: 800, height: 600, scale: 1, scrollLeft: 0, scrollTop: 0 },
+ },
+ };
+
+ await expect(startBrowserRecording("recording-tab")).rejects.toBeInstanceOf(
+ BrowserRecordingRequiresVisibleTabError,
+ );
+
+ expect(startScreencast).not.toHaveBeenCalled();
+ expect(registrySet).not.toHaveBeenCalled();
+ });
+
+ it("does not report success for a second start while the first is still starting", async () => {
+ let finishStartingScreencast: (() => void) | undefined;
+ startScreencast.mockImplementationOnce(async () => {
+ events.push("start-screencast");
+ await new Promise<void>((resolve) => {
+ finishStartingScreencast = resolve;
+ });
+ });
+
+ const firstStart = startBrowserRecording("recording-tab");
+ await vi.waitFor(() => expect(startScreencast).toHaveBeenCalledOnce());
+
+ await expect(startBrowserRecording("recording-tab")).rejects.toBeInstanceOf(
+ BrowserRecordingConflictError,
+ );
+
+ finishStartingScreencast?.();
+ await firstStart;
+ await stopBrowserRecording("recording-tab");
+ });
+
+ it("does not report success for a start while the recording is stopping", async () => {
+ let finishStoppingScreencast: (() => void) | undefined;
+ stopScreencast.mockImplementationOnce(async () => {
+ await new Promise<void>((resolve) => {
+ finishStoppingScreencast = resolve;
+ });
+ return undefined;
+ });
+
+ await startBrowserRecording("recording-tab");
+ const stopPromise = stopBrowserRecording("recording-tab");
+ await vi.waitFor(() => expect(stopScreencast).toHaveBeenCalledOnce());
+
+ await expect(startBrowserRecording("recording-tab")).rejects.toBeInstanceOf(
+ BrowserRecordingConflictError,
+ );
+
+ finishStoppingScreencast?.();
+ await stopPromise;
+ });
+
+ it("shares an in-progress stop with duplicate callers", async () => {
+ let finishStoppingScreencast: (() => void) | undefined;
+ stopScreencast.mockImplementationOnce(async () => {
+ await new Promise<void>((resolve) => {
+ finishStoppingScreencast = resolve;
+ });
+ return undefined;
+ });
+
+ await startBrowserRecording("recording-tab");
+ const firstStop = stopBrowserRecording("recording-tab");
+ await vi.waitFor(() => expect(stopScreencast).toHaveBeenCalledOnce());
+ const duplicateStop = stopBrowserRecording("recording-tab");
+
+ finishStoppingScreencast?.();
+ const [firstArtifact, duplicateArtifact] = await Promise.all([firstStop, duplicateStop]);
+
+ expect(duplicateArtifact).toEqual(firstArtifact);
+ expect(stopScreencast).toHaveBeenCalledOnce();
+ expect(save).toHaveBeenCalledOnce();
+ });
+
+ it("stops a screencast that finishes starting after cancellation", async () => {
+ let finishStartingScreencast: (() => void) | undefined;
+ startScreencast.mockImplementationOnce(async () => {
+ events.push("start-screencast");
+ await new Promise<void>((resolve) => {
+ finishStartingScreencast = resolve;
+ });
+ });
+
+ const startPromise = startBrowserRecording("recording-tab");
+ const rejectedStart = expect(startPromise).rejects.toBeInstanceOf(
+ BrowserRecordingOperationError,
+ );
+ await vi.waitFor(() => expect(startScreencast).toHaveBeenCalledOnce());
+
+ const stopPromise = stopBrowserRecording("recording-tab");
+ await vi.waitFor(() => expect(stopScreencast).toHaveBeenCalledOnce());
+ finishStartingScreencast?.();
+
+ await rejectedStart;
+ await stopPromise;
+ expect(stopScreencast).toHaveBeenCalledTimes(2);
+ expect(events.at(-1)).toBe("clear");
+ });
+
+ it("does not release the recording slot until a cancelled start settles", async () => {
+ let finishStartingScreencast: (() => void) | undefined;
+ startScreencast.mockImplementationOnce(async () => {
+ events.push("start-screencast");
+ await new Promise<void>((resolve) => {
+ finishStartingScreencast = resolve;
+ });
+ });
+
+ const firstStart = startBrowserRecording("recording-tab");
+ const rejectedFirstStart = expect(firstStart).rejects.toBeInstanceOf(
+ BrowserRecordingOperationError,
+ );
+ await vi.waitFor(() => expect(startScreencast).toHaveBeenCalledOnce());
+
+ const stopPromise = stopBrowserRecording("recording-tab");
+ const restartAfterStop = stopPromise.then(() => startBrowserRecording("recording-tab"));
+ await new Promise((resolve) => setTimeout(resolve, 0));
+ const startCallsBeforeFirstSettled = startScreencast.mock.calls.length;
+
+ finishStartingScreencast?.();
+ await rejectedFirstStart;
+ await stopPromise;
+ await restartAfterStop;
+ await stopBrowserRecording("recording-tab");
+
+ expect(startCallsBeforeFirstSettled).toBe(1);
+ });
+
+ it("fails a stop that waits too long for startup without freeing the recording slot", async () => {
+ vi.useFakeTimers();
+ let finishStartingScreencast: (() => void) | undefined;
+ startScreencast.mockImplementationOnce(async () => {
+ events.push("start-screencast");
+ await new Promise<void>((resolve) => {
+ finishStartingScreencast = resolve;
+ });
+ });
+
+ const startPromise = startBrowserRecording("recording-tab");
+ const rejectedStart = expect(startPromise).rejects.toBeInstanceOf(
+ BrowserRecordingOperationError,
+ );
+ expect(startScreencast).toHaveBeenCalledOnce();
+
+ const stopPromise = stopBrowserRecording("recording-tab");
+ await Promise.resolve();
+ await Promise.resolve();
+ expect(stopScreencast).toHaveBeenCalledOnce();
+
+ const rejection = expect(stopPromise).rejects.toMatchObject({
+ operation: "wait-startup",
+ tabId: "recording-tab",
+ });
+ await vi.advanceTimersByTimeAsync(BROWSER_RECORDING_STARTUP_SETTLE_TIMEOUT_MS);
+
+ await rejection;
+ expect(save).not.toHaveBeenCalled();
+ await expect(startBrowserRecording("recording-tab")).rejects.toBeInstanceOf(
+ BrowserRecordingConflictError,
+ );
+
+ finishStartingScreencast?.();
+ await rejectedStart;
+ const cleanupResult = await stopBrowserRecording("recording-tab");
+ expect(cleanupResult).toBeNull();
+ expect(save).not.toHaveBeenCalled();
+ expect(events.at(-1)).toBe("clear");
+ });
+});
diff --git a/apps/web/src/browser/browserRecording.ts b/apps/web/src/browser/browserRecording.ts
index f45806288..01da1e4a3 100644
--- a/apps/web/src/browser/browserRecording.ts
+++ b/apps/web/src/browser/browserRecording.ts
@@ -46,6 +46,17 @@ export class BrowserRecordingCanvasUnavailableError extends Schema.TaggedErrorCl
}
}
+export class BrowserRecordingRequiresVisibleTabError extends Schema.TaggedErrorClass<BrowserRecordingRequiresVisibleTabError>()(
+ "BrowserRecordingRequiresVisibleTabError",
+ {
+ tabId: Schema.String,
+ },
+) {
+ override get message(): string {
+ return `Browser recording requires tab ${this.tabId} to be visible.`;
+ }
+}
+
export class BrowserRecordingOperationError extends Schema.TaggedErrorClass<BrowserRecordingOperationError>()(
"BrowserRecordingOperationError",
{
@@ -55,6 +66,7 @@ export class BrowserRecordingOperationError extends Schema.TaggedErrorClass<Brow
"start-media-recorder",
"start-screencast",
"stop-screencast",
+ "wait-startup",
"stop-media-recorder",
"save-artifact",
"cleanup",
@@ -68,6 +80,16 @@ export class BrowserRecordingOperationError extends Schema.TaggedErrorClass<Brow
}
}
+const isBrowserRecordingOperationError = Schema.is(BrowserRecordingOperationError);
+
+type BrowserRecordingLifecycle =
+ | { readonly phase: "starting" }
+ | { readonly phase: "recording" }
+ | {
+ readonly phase: "stopping";
+ readonly stopPromise: Promise<DesktopPreviewRecordingArtifact | null>;
+ };
+
interface ActiveRecording {
readonly tabId: string;
readonly canvas: HTMLCanvasElement;
@@ -76,6 +98,8 @@ interface ActiveRecording {
readonly chunks: Blob[];
readonly mimeType: string;
readonly startedAt: string;
+ readonly startupSettled: Promise<void>;
+ lifecycle: BrowserRecordingLifecycle;
}
const activeBrowserRecordingTabIdAtom = Atom.make<string | null>(null).pipe(
@@ -90,6 +114,8 @@ export function useActiveBrowserRecordingTabId(): string | null {
let active: ActiveRecording | null = null;
let unsubscribeFrames: (() => void) | null = null;
+export const BROWSER_RECORDING_STARTUP_SETTLE_TIMEOUT_MS = 5_000;
+
export function readActiveBrowserRecordingTabId(): string | null {
return active?.tabId ?? null;
}
@@ -131,17 +157,58 @@ const clearActiveRecording = (recording: ActiveRecording): void => {
appAtomRegistry.set(activeBrowserRecordingTabIdAtom, null);
};
+const recordingStartupCancelledError = (
+ recording: ActiveRecording,
+ cause: unknown = new Error(`Browser recording startup was cancelled for tab ${recording.tabId}.`),
+): BrowserRecordingOperationError =>
+ new BrowserRecordingOperationError({
+ operation: "start-screencast",
+ tabId: recording.tabId,
+ cause,
+ });
+
+const isRecordingStarting = (recording: ActiveRecording): boolean =>
+ active === recording && recording.lifecycle.phase === "starting";
+
+const waitForRecordingStartupToSettle = async (recording: ActiveRecording): Promise<void> => {
+ let timeout: ReturnType<typeof setTimeout> | null = null;
+ try {
+ await Promise.race([
+ recording.startupSettled,
+ new Promise<void>((_, reject) => {
+ timeout = setTimeout(() => {
+ reject(new Error(`Browser recording startup did not settle for tab ${recording.tabId}.`));
+ }, BROWSER_RECORDING_STARTUP_SETTLE_TIMEOUT_MS);
+ }),
+ ]);
+ } catch (cause) {
+ throw new BrowserRecordingOperationError({
+ operation: "wait-startup",
+ tabId: recording.tabId,
+ cause,
+ });
+ } finally {
+ if (timeout !== null) clearTimeout(timeout);
+ }
+};
+
+const isStartupWaitTimeout = (error: unknown): error is BrowserRecordingOperationError =>
+ isBrowserRecordingOperationError(error) && error.operation === "wait-startup";
+
export async function startBrowserRecording(tabId: string): Promise<string> {
const bridge = previewBridge;
if (!bridge) throw new BrowserRecordingUnavailableError({ tabId });
if (active) {
- if (active.tabId === tabId) return active.startedAt;
+ if (active.tabId === tabId && active.lifecycle.phase === "recording") {
+ return active.startedAt;
+ }
throw new BrowserRecordingConflictError({
requestedTabId: tabId,
activeTabId: active.tabId,
});
}
const surface = useBrowserSurfaceStore.getState().byTabId[tabId];
+ if (!surface?.visible) throw new BrowserRecordingRequiresVisibleTabError({ tabId });
const recordingSize = surface?.content ?? surface?.rect;
const canvas = document.createElement("canvas");
canvas.width = Math.max(1, recordingSize?.width ?? 1280);
@@ -171,65 +238,104 @@ export async function startBrowserRecording(tabId: string): Promise<string> {
}
const startedAt = new Date().toISOString();
const chunks: Blob[] = [];
+ let settleStartup: (() => void) | undefined;
+ const startupSettled = new Promise<void>((resolve) => {
+ settleStartup = resolve;
+ });
recorder.addEventListener("dataavailable", (event) => {
if (event.data.size > 0) chunks.push(event.data);
});
- const recording = { tabId, canvas, context, recorder, chunks, mimeType, startedAt };
+ const recording: ActiveRecording = {
+ tabId,
+ canvas,
+ context,
+ recorder,
+ chunks,
+ mimeType,
+ startedAt,
+ startupSettled,
+ lifecycle: { phase: "starting" },
+ };
active = recording;
try {
- unsubscribeFrames ??= bridge.recording.onFrame(drawFrame);
- } catch (cause) {
- clearActiveRecording(recording);
- throw new BrowserRecordingOperationError({
- operation: "subscribe-frames",
- tabId,
- cause,
- });
- }
- try {
- recorder.start(1_000);
- } catch (cause) {
- clearActiveRecording(recording);
- throw new BrowserRecordingOperationError({
- operation: "start-media-recorder",
- tabId,
- cause,
- });
- }
- try {
- await bridge.recording.startScreencast(tabId);
- } catch (cause) {
- let cleanupCause: unknown;
try {
- await stopMediaRecorder(recorder);
- } catch (error) {
- cleanupCause = error;
- } finally {
+ unsubscribeFrames ??= bridge.recording.onFrame(drawFrame);
+ } catch (cause) {
clearActiveRecording(recording);
+ throw new BrowserRecordingOperationError({
+ operation: "subscribe-frames",
+ tabId,
+ cause,
+ });
}
- throw new BrowserRecordingOperationError({
- operation: "start-screencast",
- tabId,
- cause:
- cleanupCause === undefined
- ? cause
- : new AggregateError(
- [cause, cleanupCause],
- `Browser recording start and cleanup failed for tab ${tabId}.`,
- { cause },
- ),
- });
+ try {
+ recorder.start(1_000);
+ } catch (cause) {
+ clearActiveRecording(recording);
+ throw new BrowserRecordingOperationError({
+ operation: "start-media-recorder",
+ tabId,
+ cause,
+ });
+ }
+ if (!isRecordingStarting(recording)) {
+ throw recordingStartupCancelledError(recording);
+ }
+ try {
+ await bridge.recording.startScreencast(tabId);
+ } catch (cause) {
+ if (!isRecordingStarting(recording)) {
+ throw recordingStartupCancelledError(recording, cause);
+ }
+ let cleanupCause: unknown;
+ try {
+ await stopMediaRecorder(recorder);
+ } catch (error) {
+ cleanupCause = error;
+ } finally {
+ clearActiveRecording(recording);
+ }
+ throw new BrowserRecordingOperationError({
+ operation: "start-screencast",
+ tabId,
+ cause:
+ cleanupCause === undefined
+ ? cause
+ : new AggregateError(
+ [cause, cleanupCause],
+ `Browser recording start and cleanup failed for tab ${tabId}.`,
+ { cause },
+ ),
+ });
+ }
+ if (!isRecordingStarting(recording)) {
+ try {
+ await bridge.recording.stopScreencast(tabId);
+ } catch (cause) {
+ throw recordingStartupCancelledError(
+ recording,
+ new AggregateError(
+ [new Error(`Browser recording startup was cancelled for tab ${tabId}.`), cause],
+ `Browser recording startup cancellation failed for tab ${tabId}.`,
+ { cause },
+ ),
+ );
+ }
+ throw recordingStartupCancelledError(recording);
+ }
+ recording.lifecycle = { phase: "recording" };
+ appAtomRegistry.set(activeBrowserRecordingTabIdAtom, tabId);
+ return startedAt;
+ } finally {
+ settleStartup?.();
}
- appAtomRegistry.set(activeBrowserRecordingTabIdAtom, tabId);
- return startedAt;
}
-export async function stopBrowserRecording(
- tabId: string,
-): Promise<DesktopPreviewRecordingArtifact | null> {
- const bridge = previewBridge;
- const recording = active;
- if (!bridge || !recording || recording.tabId !== tabId) return null;
+const finalizeBrowserRecording = async (
+ bridge: NonNullable<typeof previewBridge>,
+ recording: ActiveRecording,
+): Promise<DesktopPreviewRecordingArtifact> => {
+ const { tabId } = recording;
let result:
| { readonly _tag: "Success"; readonly artifact: DesktopPreviewRecordingArtifact }
| { readonly _tag: "Failure"; readonly error: unknown };
@@ -243,6 +349,7 @@ export async function stopBrowserRecording(
cause,
});
}
+ await waitForRecordingStartupToSettle(recording);
try {
await stopMediaRecorder(recording.recorder);
} catch (cause) {
@@ -271,6 +378,14 @@ export async function stopBrowserRecording(
result = { _tag: "Failure", error };
}
+ if (result._tag === "Failure" && isStartupWaitTimeout(result.error)) {
+ // Do not clear `active` yet. The renderer-side start promise can still
+ // resolve later, and its cancellation path will call `stopScreencast`.
+ // Keeping the slot reserved prevents a newer recording for this tab from
+ // being started and then accidentally stopped by the older late cleanup.
+ throw result.error;
+ }
+
let cleanupError: BrowserRecordingOperationError | undefined;
try {
await stopMediaRecorder(recording.recorder);
@@ -300,4 +415,41 @@ export async function stopBrowserRecording(
}
if (cleanupError) throw cleanupError;
return result.artifact;
+};
+
+const discardBrowserRecording = async (
+ bridge: NonNullable<typeof previewBridge>,
+ recording: ActiveRecording,
+): Promise<null> => {
+ try {
+ await bridge.recording.stopScreencast(recording.tabId).catch(() => undefined);
+ await stopMediaRecorder(recording.recorder).catch(() => undefined);
+ return null;
+ } finally {
+ clearActiveRecording(recording);
+ }
+};
+
+export function stopBrowserRecording(
+ tabId: string,
+): Promise<DesktopPreviewRecordingArtifact | null> {
+ const bridge = previewBridge;
+ const recording = active;
+ if (!bridge || !recording || recording.tabId !== tabId) return Promise.resolve(null);
+ if (recording.lifecycle.phase === "stopping") return recording.lifecycle.stopPromise;
+
+ const stopPromise = Promise.resolve()
+ .then(() => finalizeBrowserRecording(bridge, recording))
+ .catch((error) => {
+ if (isStartupWaitTimeout(error) && active === recording) {
+ const cleanupAfterStartup = recording.startupSettled.then(() =>
+ discardBrowserRecording(bridge, recording),
+ );
+ recording.lifecycle = { phase: "stopping", stopPromise: cleanupAfterStartup };
+ void cleanupAfterStartup.catch(() => undefined);
+ }
+ throw error;
+ });
+ recording.lifecycle = { phase: "stopping", stopPromise };
+ return stopPromise;
}
diff --git a/apps/web/src/browser/browserRecordingScope.test.ts b/apps/web/src/browser/browserRecordingScope.test.ts
index ae91f1a7f..6b3adf221 100644
--- a/apps/web/src/browser/browserRecordingScope.test.ts
+++ b/apps/web/src/browser/browserRecordingScope.test.ts
@@ -3,9 +3,14 @@ import { describe, expect, it } from "vite-plus/test";
import { resolveBrowserRecordingStopTarget } from "./browserRecordingScope";
describe("resolveBrowserRecordingStopTarget", () => {
- it("stops the active recording even after the requested tab changes", () => {
+ it("stops the active recording when no explicit tab was requested", () => {
expect(resolveBrowserRecordingStopTarget("tab-a")).toBe("tab-a");
expect(resolveBrowserRecordingStopTarget("tab-b")).toBe("tab-b");
expect(resolveBrowserRecordingStopTarget(null)).toBeNull();
});
+
+ it("only stops an explicitly requested tab when it owns the recording", () => {
+ expect(resolveBrowserRecordingStopTarget("tab-a", "tab-a")).toBe("tab-a");
+ expect(resolveBrowserRecordingStopTarget("tab-a", "tab-b")).toBeNull();
+ });
});
diff --git a/apps/web/src/browser/browserRecordingScope.ts b/apps/web/src/browser/browserRecordingScope.ts
index 807b5d0c9..92f58016a 100644
--- a/apps/web/src/browser/browserRecordingScope.ts
+++ b/apps/web/src/browser/browserRecordingScope.ts
@@ -1,3 +1,7 @@
-export function resolveBrowserRecordingStopTarget(activeTabId: string | null): string | null {
- return activeTabId;
+export function resolveBrowserRecordingStopTarget(
+ activeTabId: string | null,
+ requestedTabId?: string,
+): string | null {
+ if (activeTabId === null) return null;
+ return requestedTabId === undefined || requestedTabId === activeTabId ? activeTabId : null;
}
diff --git a/apps/web/src/browser/browserSurfaceStore.test.ts b/apps/web/src/browser/browserSurfaceStore.test.ts
index 1445303b8..ecfce8cb4 100644
--- a/apps/web/src/browser/browserSurfaceStore.test.ts
+++ b/apps/web/src/browser/browserSurfaceStore.test.ts
@@ -1,8 +1,16 @@
-import { describe, expect, it } from "vite-plus/test";
+import { beforeEach, describe, expect, it } from "vite-plus/test";
-import { resolveBrowserSurfacePanelRect, useBrowserSurfaceStore } from "./browserSurfaceStore";
+import {
+ acquireBrowserSurface,
+ resolveBrowserSurfacePanelRect,
+ useBrowserSurfaceStore,
+} from "./browserSurfaceStore";
describe("browserSurfaceStore", () => {
+ beforeEach(() => {
+ useBrowserSurfaceStore.setState({ byTabId: {} });
+ });
+
it("tracks content dimensions for a browser that has never been visible", () => {
const tabId = "hidden-browser-surface-content-test";
useBrowserSurfaceStore.getState().presentContent(tabId, {
@@ -28,11 +36,43 @@ describe("browserSurfaceStore", () => {
expect(
resolveBrowserSurfacePanelRect(
{
- hidden: { rect: staleRect, visible: false, content: null, updatedAt: 1 },
- active: { rect: liveRect, visible: true, content: null, updatedAt: 2 },
+ hidden: { rect: staleRect, visible: false, content: null, updatedAt: 1, owner: null },
+ active: { rect: liveRect, visible: true, content: null, updatedAt: 2, owner: null },
},
"hidden",
),
).toEqual(liveRect);
});
+
+ it("ignores updates and releases from a stale surface lease", () => {
+ const tabId = "leased-browser-surface";
+ const staleRect = { x: 0, y: 0, width: 500, height: 700 };
+ const liveRect = { x: 10, y: 20, width: 900, height: 640 };
+ const staleLease = acquireBrowserSurface(tabId);
+ staleLease.present(staleRect, true);
+
+ const liveLease = acquireBrowserSurface(tabId);
+ liveLease.present(liveRect, true);
+ staleLease.present(staleRect, true);
+ staleLease.release();
+
+ expect(useBrowserSurfaceStore.getState().byTabId[tabId]).toMatchObject({
+ rect: liveRect,
+ visible: true,
+ });
+ });
+
+ it("hides a surface when its current lease is released", () => {
+ const tabId = "released-browser-surface";
+ const lease = acquireBrowserSurface(tabId);
+ lease.present({ x: 10, y: 20, width: 900, height: 640 }, true);
+
+ lease.release();
+ lease.present({ x: 0, y: 0, width: 1, height: 1 }, true);
+
+ expect(useBrowserSurfaceStore.getState().byTabId[tabId]).toMatchObject({
+ visible: false,
+ owner: null,
+ });
+ });
});
diff --git a/apps/web/src/browser/browserSurfaceStore.ts b/apps/web/src/browser/browserSurfaceStore.ts
index 79095f0bd..58012a11a 100644
--- a/apps/web/src/browser/browserSurfaceStore.ts
+++ b/apps/web/src/browser/browserSurfaceStore.ts
@@ -12,6 +12,7 @@ export interface BrowserSurfacePresentation {
readonly visible: boolean;
readonly content: BrowserSurfaceContentPresentation | null;
readonly updatedAt: number;
+ readonly owner: symbol | null;
}
export interface BrowserSurfaceContentPresentation {
@@ -26,9 +27,20 @@ export interface BrowserSurfaceContentPresentation {
interface BrowserSurfaceStoreState {
readonly byTabId: Record<string, BrowserSurfacePresentation>;
- readonly present: (tabId: string, rect: BrowserSurfaceRect, visible: boolean) => void;
+ readonly claim: (tabId: string, owner: symbol) => void;
+ readonly present: (
+ tabId: string,
+ owner: symbol,
+ rect: BrowserSurfaceRect,
+ visible: boolean,
+ ) => void;
readonly presentContent: (tabId: string, content: BrowserSurfaceContentPresentation) => void;
- readonly hide: (tabId: string) => void;
+ readonly release: (tabId: string, owner: symbol) => void;
+}
+
+export interface BrowserSurfaceLease {
+ readonly present: (rect: BrowserSurfaceRect, visible: boolean) => void;
+ readonly release: () => void;
}
export function resolveBrowserSurfacePanelRect(
@@ -60,14 +72,32 @@ const rectEquals = (left: BrowserSurfaceRect | null, right: BrowserSurfaceRect):
export const useBrowserSurfaceStore = create<BrowserSurfaceStoreState>()((set) => ({
byTabId: {},
- present: (tabId, rect, visible) =>
+ claim: (tabId, owner) =>
+ set((state) => {
+ const current = state.byTabId[tabId];
+ if (current?.owner === owner) return state;
+ return {
+ byTabId: {
+ ...state.byTabId,
+ [tabId]: {
+ rect: current?.rect ?? null,
+ visible: false,
+ content: current?.content ?? null,
+ updatedAt: Date.now(),
+ owner,
+ },
+ },
+ };
+ }),
+ present: (tabId, owner, rect, visible) =>
set((state) => {
const current = state.byTabId[tabId];
+ if (current?.owner !== owner) return state;
if (current && current.visible === visible && rectEquals(current.rect, rect)) return state;
return {
byTabId: {
...state.byTabId,
- [tabId]: { rect, visible, content: current?.content ?? null, updatedAt: Date.now() },
+ [tabId]: { ...current, rect, visible, updatedAt: Date.now() },
},
};
}),
@@ -83,6 +113,7 @@ export const useBrowserSurfaceStore = create<BrowserSurfaceStoreState>()((set) =
visible: false,
content,
updatedAt: Date.now(),
+ owner: null,
},
},
};
@@ -107,15 +138,33 @@ export const useBrowserSurfaceStore = create<BrowserSurfaceStoreState>()((set) =
},
};
}),
- hide: (tabId) =>
+ release: (tabId, owner) =>
set((state) => {
const current = state.byTabId[tabId];
- if (!current || !current.visible) return state;
+ if (current?.owner !== owner) return state;
return {
byTabId: {
...state.byTabId,
- [tabId]: { ...current, visible: false, updatedAt: Date.now() },
+ [tabId]: { ...current, visible: false, updatedAt: Date.now(), owner: null },
},
};
}),
}));
+
+export function acquireBrowserSurface(tabId: string): BrowserSurfaceLease {
+ const owner = Symbol(`browser-surface:${tabId}`);
+ let released = false;
+ useBrowserSurfaceStore.getState().claim(tabId, owner);
+
+ return {
+ present: (rect, visible) => {
+ if (released) return;
+ useBrowserSurfaceStore.getState().present(tabId, owner, rect, visible);
+ },
+ release: () => {
+ if (released) return;
+ released = true;
+ useBrowserSurfaceStore.getState().release(tabId, owner);
+ },
+ };
+}
diff --git a/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts b/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts
new file mode 100644
index 000000000..826684bb0
--- /dev/null
+++ b/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts
@@ -0,0 +1,43 @@
+import { describe, expect, it } from "vite-plus/test";
+
+import {
+ HIDDEN_BROWSER_WEBVIEW_OFFSET,
+ resolveHostedBrowserWebviewWrapperStyle,
+} from "./hostedBrowserWebviewStyle";
+
+describe("resolveHostedBrowserWebviewWrapperStyle", () => {
+ it("places an active webview on its presented surface", () => {
+ expect(
+ resolveHostedBrowserWebviewWrapperStyle({
+ active: true,
+ rect: { x: 12, y: 34, width: 800, height: 600 },
+ hiddenSize: { width: 1280, height: 800 },
+ }),
+ ).toEqual({
+ left: 12,
+ top: 34,
+ width: 800,
+ height: 600,
+ zIndex: 30,
+ pointerEvents: "auto",
+ });
+ });
+
+ it("keeps an inactive webview paintable while moving it offscreen", () => {
+ const style = resolveHostedBrowserWebviewWrapperStyle({
+ active: false,
+ rect: { x: 12, y: 34, width: 800, height: 600 },
+ hiddenSize: { width: 393, height: 852 },
+ });
+
+ expect(style).toEqual({
+ left: HIDDEN_BROWSER_WEBVIEW_OFFSET,
+ top: HIDDEN_BROWSER_WEBVIEW_OFFSET,
+ width: 393,
+ height: 852,
+ zIndex: -1,
+ pointerEvents: "none",
+ visibility: "visible",
+ });
+ });
+});
diff --git a/apps/web/src/browser/hostedBrowserWebviewStyle.ts b/apps/web/src/browser/hostedBrowserWebviewStyle.ts
new file mode 100644
index 000000000..4dade986e
--- /dev/null
+++ b/apps/web/src/browser/hostedBrowserWebviewStyle.ts
@@ -0,0 +1,49 @@
+import type { BrowserSurfaceRect } from "./browserSurfaceStore";
+
+export interface HostedBrowserWebviewSize {
+ readonly width: number;
+ readonly height: number;
+}
+
+export interface HostedBrowserWebviewWrapperStyle {
+ readonly left: number;
+ readonly top: number;
+ readonly width: number;
+ readonly height: number;
+ readonly zIndex: number;
+ readonly pointerEvents: "auto" | "none";
+ readonly visibility?: "visible";
+}
+
+export const HIDDEN_BROWSER_WEBVIEW_OFFSET = -100_000;
+
+export function resolveHostedBrowserWebviewWrapperStyle(input: {
+ readonly active: boolean;
+ readonly rect: BrowserSurfaceRect | null;
+ readonly hiddenSize: HostedBrowserWebviewSize;
+}): HostedBrowserWebviewWrapperStyle {
+ const { active, hiddenSize, rect } = input;
+ if (active && rect) {
+ return {
+ left: rect.x,
+ top: rect.y,
+ width: rect.width,
+ height: rect.height,
+ zIndex: 30,
+ pointerEvents: "auto",
+ };
+ }
+
+ return {
+ left: HIDDEN_BROWSER_WEBVIEW_OFFSET,
+ top: HIDDEN_BROWSER_WEBVIEW_OFFSET,
+ width: hiddenSize.width,
+ height: hiddenSize.height,
+ zIndex: -1,
+ pointerEvents: "none",
+ // Keep the guest CSS-visible even while physically offscreen. Electron
+ // webviews can keep metadata/status alive under `visibility:hidden` while
+ // CDP Runtime/Input commands stall, which breaks offscreen automation.
+ visibility: "visible",
+ };
+}
diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx
index 94d8c13a3..d637611a7 100644
--- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx
+++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx
@@ -56,6 +56,7 @@ import { createPreviewAutomationRequestConsumerAtom } from "./previewAutomationR
import { createPreviewAutomationClientId } from "./previewAutomationClientId";
import {
needsPreviewAutomationSessionSync,
+ resolvePreviewAutomationOpenTab,
resolvePreviewAutomationTarget,
} from "./previewAutomationTarget";
import { isPreviewViewportReady } from "./previewViewportReadiness";
@@ -230,6 +231,16 @@ const currentStatus = async (
};
};
+const raiseAtomCommandFailure = (result: Parameters<typeof squashAtomCommandFailure>[0]): never => {
+ throw squashAtomCommandFailure(result);
+};
+
+const raisePreviewAutomationHostError = (
+ error: PreviewAutomationRecordingNotActiveError,
+): never => {
+ throw error;
+};
+
export function PreviewAutomationHosts() {
const { environments } = useEnvironments();
if (!isElectron || !previewBridge?.automation) return null;
@@ -304,7 +315,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId })
registry.refresh(previewEnvironment.list(listTarget));
const result = await listPreviews(listTarget);
if (result._tag === "Failure") {
- throw squashAtomCommandFailure(result);
+ return raiseAtomCommandFailure(result);
}
reconcilePreviewServerSessions(threadRef, result.value.sessions);
state = readThreadPreviewState(threadRef);
@@ -332,8 +343,11 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId })
return await currentStatus(threadRef, tabId);
case "open": {
const input = request.input as PreviewAutomationOpenInput;
- let activeTabId =
- (input.reuseExistingTab ?? true) ? (state.snapshot?.tabId ?? null) : null;
+ let activeTabId = resolvePreviewAutomationOpenTab(
+ state,
+ request.tabId,
+ input.reuseExistingTab ?? true,
+ );
let activeSnapshot = activeTabId
? (state.sessions[activeTabId] ?? state.snapshot ?? undefined)
: undefined;
@@ -348,7 +362,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId })
},
});
if (result._tag === "Failure") {
- throw squashAtomCommandFailure(result);
+ return raiseAtomCommandFailure(result);
}
const snapshot = result.value;
applyPreviewServerSnapshot(threadRef, snapshot);
@@ -416,7 +430,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId })
},
});
if (result._tag === "Failure") {
- throw squashAtomCommandFailure(result);
+ return raiseAtomCommandFailure(result);
}
updatePreviewServerSnapshot(threadRef, result.value);
const viewport = await waitForRenderedViewport(
@@ -492,15 +506,20 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId })
}
case "recordingStop": {
const recordingTabId = readActiveBrowserRecordingTabId();
- const stopTabId = resolveBrowserRecordingStopTarget(recordingTabId);
+ const stopTabId = resolveBrowserRecordingStopTarget(
+ recordingTabId,
+ request.tabIdExplicit ? request.tabId : undefined,
+ );
const artifact = stopTabId ? await stopBrowserRecording(stopTabId) : null;
if (!artifact) {
- throw new PreviewAutomationRecordingNotActiveError({
- requestId: request.requestId,
- environmentId,
- threadId: request.threadId,
- tabId,
- });
+ return raisePreviewAutomationHostError(
+ new PreviewAutomationRecordingNotActiveError({
+ requestId: request.requestId,
+ environmentId,
+ threadId: request.threadId,
+ tabId,
+ }),
+ );
}
return artifact;
}
diff --git a/apps/web/src/components/preview/previewAutomationTarget.test.ts b/apps/web/src/components/preview/previewAutomationTarget.test.ts
index 379e35190..11a70951b 100644
--- a/apps/web/src/components/preview/previewAutomationTarget.test.ts
+++ b/apps/web/src/components/preview/previewAutomationTarget.test.ts
@@ -3,6 +3,7 @@ import { describe, expect, it } from "vite-plus/test";
import {
needsPreviewAutomationSessionSync,
+ resolvePreviewAutomationOpenTab,
resolvePreviewAutomationTarget,
} from "./previewAutomationTarget";
@@ -42,4 +43,17 @@ describe("preview automation target selection", () => {
),
).toEqual({ tabId: null, snapshot: null });
});
+
+ it("reuses the provider session's pinned tab instead of the mutable UI tab", () => {
+ const uiActive = snapshot("tab-ui-active");
+ const agentTab = snapshot("tab-opened-by-agent");
+ const state = {
+ snapshot: uiActive,
+ sessions: { [uiActive.tabId]: uiActive, [agentTab.tabId]: agentTab },
+ };
+
+ expect(resolvePreviewAutomationOpenTab(state, agentTab.tabId, true)).toBe(agentTab.tabId);
+ expect(resolvePreviewAutomationOpenTab(state, undefined, true)).toBe(uiActive.tabId);
+ expect(resolvePreviewAutomationOpenTab(state, agentTab.tabId, false)).toBeNull();
+ });
});
diff --git a/apps/web/src/components/preview/previewAutomationTarget.ts b/apps/web/src/components/preview/previewAutomationTarget.ts
index 1dc9f17f7..f1858f87d 100644
--- a/apps/web/src/components/preview/previewAutomationTarget.ts
+++ b/apps/web/src/components/preview/previewAutomationTarget.ts
@@ -23,3 +23,15 @@ export function resolvePreviewAutomationTarget(
const snapshot = requestedTabId ? (state.sessions[requestedTabId] ?? null) : state.snapshot;
return { tabId: snapshot?.tabId ?? null, snapshot };
}
+
+export function resolvePreviewAutomationOpenTab(
+ state: PreviewAutomationSessionIndex,
+ requestedTabId: string | undefined,
+ reuseExistingTab: boolean,
+): string | null {
+ if (!reuseExistingTab) return null;
+ if (requestedTabId !== undefined) {
+ return state.sessions[requestedTabId]?.tabId ?? null;
+ }
+ return state.snapshot?.tabId ?? null;
+}
diff --git a/packages/contracts/src/preview.test.ts b/packages/contracts/src/preview.test.ts
index 800447938..e2e8336cd 100644
--- a/packages/contracts/src/preview.test.ts
+++ b/packages/contracts/src/preview.test.ts
@@ -11,6 +11,7 @@ import {
import {
PreviewAutomationHost,
PreviewAutomationError,
+ PreviewAutomationOpenInput,
PreviewAutomationResizeInput,
PreviewAutomationResizeResult,
PreviewAutomationStatus,
@@ -22,6 +23,7 @@ const decodeNavStatus = Schema.decodeUnknownSync(PreviewNavStatus);
const decodeServer = Schema.decodeUnknownSync(DiscoveredLocalServer);
const decodeViewport = Schema.decodeUnknownSync(PreviewViewportSetting);
const decodeResizeInput = Schema.decodeUnknownSync(PreviewAutomationResizeInput);
+const decodeOpenInput = Schema.decodeUnknownSync(PreviewAutomationOpenInput);
const decodeResizeResult = Schema.decodeUnknownSync(PreviewAutomationResizeResult);
const decodeAutomationHost = Schema.decodeUnknownSync(PreviewAutomationHost);
const decodeAutomationError = Schema.decodeUnknownSync(PreviewAutomationError);
@@ -128,6 +130,20 @@ describe("PreviewAutomationResizeInput", () => {
});
});
+describe("preview automation tab targeting", () => {
+ it("accepts an explicit tab and rejects contradictory open behavior", () => {
+ expect(decodeResizeInput({ tabId: "tab-app", mode: "fill" })).toMatchObject({
+ tabId: "tab-app",
+ mode: "fill",
+ });
+ expect(decodeOpenInput({ tabId: "tab-app", reuseExistingTab: true })).toMatchObject({
+ tabId: "tab-app",
+ reuseExistingTab: true,
+ });
+ expect(() => decodeOpenInput({ tabId: "tab-app", reuseExistingTab: false })).toThrow();
+ });
+});
+
describe("PreviewAutomationHost", () => {
it("accepts legacy hosts and current operation advertisements", () => {
expect(decodeAutomationHost({ clientId: "legacy", environmentId: "environment-1" })).toEqual({
diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts
index 47dc7fc24..6431dd0dc 100644
--- a/packages/contracts/src/previewAutomation.ts
+++ b/packages/contracts/src/previewAutomation.ts
@@ -50,6 +50,21 @@ export const PREVIEW_AUTOMATION_OPERATIONS = [
export const PreviewAutomationOperation = Schema.Literals(PREVIEW_AUTOMATION_OPERATIONS);
export type PreviewAutomationOperation = typeof PreviewAutomationOperation.Type;
+const PreviewAutomationTabTargetFields = {
+ tabId: Schema.optional(
+ PreviewTabId.annotate({
+ description:
+ "Exact collaborative browser tab to target. Omit to use this agent session's current tab.",
+ }),
+ ).annotate({
+ description:
+ "Exact collaborative browser tab to target. Omit to use this agent session's current tab.",
+ }),
+};
+
+export const PreviewAutomationTabTargetInput = Schema.Struct(PreviewAutomationTabTargetFields);
+export type PreviewAutomationTabTargetInput = typeof PreviewAutomationTabTargetInput.Type;
+
export const PreviewAutomationStatus = Schema.Struct({
available: Schema.Boolean,
visible: Schema.Boolean,
@@ -65,6 +80,7 @@ export const PreviewAutomationStatus = Schema.Struct({
export type PreviewAutomationStatus = typeof PreviewAutomationStatus.Type;
export const PreviewAutomationOpenInput = Schema.Struct({
+ ...PreviewAutomationTabTargetFields,
url: Schema.optional(BoundedUrl).annotate({
description:
"Optional initial page URL, for example https://t3.chat or localhost:5173. Omit to open a blank tab.",
@@ -77,13 +93,21 @@ export const PreviewAutomationOpenInput = Schema.Struct({
reuseExistingTab: Schema.optional(
Schema.Boolean.annotate({
description:
- "Reuse the thread's active browser tab when available. Defaults to true; set false to create a new tab.",
+ "Reuse tabId when supplied, otherwise this agent session's current tab. Defaults to true; set false to create a new tab.",
}),
),
-}).annotate({
- description:
- "Opens the collaborative browser for the current thread. Use preview_navigate afterward when readiness waiting matters.",
-});
+})
+ .check(
+ Schema.makeFilter(
+ (input) =>
+ !(input.tabId !== undefined && input.reuseExistingTab === false) ||
+ "tabId cannot be combined with reuseExistingTab=false.",
+ ),
+ )
+ .annotate({
+ description:
+ "Opens the collaborative browser for the current thread. Use preview_navigate afterward when readiness waiting matters.",
+ });
export type PreviewAutomationOpenInput = typeof PreviewAutomationOpenInput.Type;
export const BrowserNavigationTarget = Schema.Union([
@@ -117,6 +141,7 @@ export const BrowserNavigationTarget = Schema.Union([
export type BrowserNavigationTarget = typeof BrowserNavigationTarget.Type;
export const PreviewAutomationNavigateInput = Schema.Struct({
+ ...PreviewAutomationTabTargetFields,
url: Schema.optional(BoundedUrl).annotate({
description:
"Website URL, for example https://t3.chat. Use this for public pages and directly reachable URLs.",
@@ -155,6 +180,7 @@ export const PreviewAutomationNavigateInput = Schema.Struct({
export type PreviewAutomationNavigateInput = typeof PreviewAutomationNavigateInput.Type;
export const PreviewAutomationResizeInput = Schema.Struct({
+ ...PreviewAutomationTabTargetFields,
mode: Schema.Literals(["fill", "freeform", "preset"]).annotate({
description:
"Viewport mode: fill follows the preview panel, freeform uses exact independently resizable dimensions, and preset uses a named device size.",
@@ -239,6 +265,7 @@ const LegacySelector = TrimmedNonEmptyString.annotate({
});
export const PreviewAutomationClickInput = Schema.Struct({
+ ...PreviewAutomationTabTargetFields,
selector: Schema.optional(LegacySelector).annotate({
description:
"Legacy CSS selector such as button[type='submit']. Prefer locator for resilient role/text targeting.",
@@ -277,6 +304,7 @@ export const PreviewAutomationClickInput = Schema.Struct({
export type PreviewAutomationClickInput = typeof PreviewAutomationClickInput.Type;
export const PreviewAutomationTypeInput = Schema.Struct({
+ ...PreviewAutomationTabTargetFields,
text: Schema.String.annotate({ description: "Literal text to insert." }),
selector: Schema.optional(LegacySelector).annotate({
description: "Legacy CSS selector for the input. Prefer locator.",
@@ -306,6 +334,7 @@ export const PreviewAutomationTypeInput = Schema.Struct({
export type PreviewAutomationTypeInput = typeof PreviewAutomationTypeInput.Type;
export const PreviewAutomationPressInput = Schema.Struct({
+ ...PreviewAutomationTabTargetFields,
key: Schema.String.check(Schema.isTrimmed())
.check(
Schema.isNonEmpty({
@@ -326,6 +355,7 @@ export const PreviewAutomationPressInput = Schema.Struct({
export type PreviewAutomationPressInput = typeof PreviewAutomationPressInput.Type;
export const PreviewAutomationScrollInput = Schema.Struct({
+ ...PreviewAutomationTabTargetFields,
deltaX: Schema.optional(
Schema.Finite.annotate({
description: "Horizontal scroll delta in CSS pixels. Positive scrolls right. Defaults to 0.",
@@ -360,6 +390,7 @@ export const PreviewAutomationScrollInput = Schema.Struct({
export type PreviewAutomationScrollInput = typeof PreviewAutomationScrollInput.Type;
export const PreviewAutomationEvaluateInput = Schema.Struct({
+ ...PreviewAutomationTabTargetFields,
expression: Schema.String.check(Schema.isTrimmed())
.check(
Schema.isNonEmpty({
@@ -388,6 +419,7 @@ export const PreviewAutomationEvaluateInput = Schema.Struct({
export type PreviewAutomationEvaluateInput = typeof PreviewAutomationEvaluateInput.Type;
export const PreviewAutomationWaitForInput = Schema.Struct({
+ ...PreviewAutomationTabTargetFields,
selector: Schema.optional(LegacySelector).annotate({
description: "Legacy CSS selector that must match an element. Prefer locator.",
}),
@@ -537,6 +569,7 @@ export const PreviewAutomationRequest = Schema.Struct({
requestId: TrimmedNonEmptyString,
threadId: ThreadId,
tabId: Schema.optional(PreviewTabId),
+ tabIdExplicit: Schema.optional(Schema.Boolean),
operation: PreviewAutomationOperation,
input: Schema.Unknown,
timeoutMs: Schema.Int.check(Schema.isGreaterThan(0)),