Skip to content

Automate browser based workflows with AI

feat(SKY-11665): studio Browser pane — lock take-control while a block run executes (#7008)

Celal Zamanoğlucddad3a3f005
diff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsx
index 3742c7b5f..28198468c 100644
--- a/skyvern-frontend/src/components/BrowserStream.tsx
+++ b/skyvern-frontend/src/components/BrowserStream.tsx
@@ -722,6 +722,15 @@ function BrowserStream({
}
}, [interactive]);
+ // When control can no longer be offered (buttons hidden and not inherently
+ // interactive), a prior grab must be released or its input keeps flowing.
+ // Recording is exempt: it holds take-control for exfiltration.
+ useEffect(() => {
+ if (!interactive && !showControlButtons && !recordingStore.isRecording) {
+ setUserIsControlling(false);
+ }
+ }, [interactive, showControlButtons, recordingStore.isRecording]);
+
const theUserIsControlling =
userIsControlling || (interactive && !showControlButtons);
diff --git a/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsx b/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsx
index e082c1cb3..d824523bb 100644
--- a/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsx
+++ b/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsx
@@ -152,6 +152,14 @@ function BrowserSessionStream({
onActivityRef.current = onActivity;
}, [onActivity]);
+ // Once control can't be offered (input socket torn down), forget any prior
+ // grab so re-enabling doesn't silently restore control without a new click.
+ useEffect(() => {
+ if (!controllable) {
+ setUserIsControlling(false);
+ }
+ }, [controllable, setUserIsControlling]);
+
useEffect(() => {
let cancelled = false;
setStreamImgSrc("");
diff --git a/skyvern-frontend/src/routes/workflows/studio/StudioBrowserStream.test.tsx b/skyvern-frontend/src/routes/workflows/studio/StudioBrowserStream.test.tsx
index d26afb05f..00e84f9ce 100644
--- a/skyvern-frontend/src/routes/workflows/studio/StudioBrowserStream.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/studio/StudioBrowserStream.test.tsx
@@ -4,6 +4,8 @@ import { fireEvent, render, screen, waitFor } from "@testing-library/react";
import { MemoryRouter, Route, Routes } from "react-router-dom";
import { beforeEach, describe, expect, it, vi } from "vitest";
+import { Status } from "@/api/types";
+import { useRecordingStore } from "@/store/useRecordingStore";
import { useStudioBrowserStore } from "@/store/useStudioBrowserStore";
import { StudioBrowserStream } from "./StudioBrowserStream";
@@ -13,12 +15,19 @@ const runtimeConfigMock = vi.hoisted(() => ({
browserStreamingMode: "vnc",
}));
+const workflowRunQueryMock = vi.hoisted(() => vi.fn());
+
vi.mock("../hooks/useDebugSessionQuery", () => ({
useDebugSessionQuery: () => ({
data: { browser_session_id: "pbs_test" },
}),
}));
+vi.mock("../hooks/useWorkflowRunWithWorkflowQuery", () => ({
+ useWorkflowRunWithWorkflowQuery: (options?: { workflowRunId?: string }) =>
+ workflowRunQueryMock(options),
+}));
+
vi.mock("@/hooks/useRuntimeConfig", () => ({
useBrowserStreamingMode: () => ({
browserStreamingMode: runtimeConfigMock.browserStreamingMode,
@@ -71,6 +80,7 @@ vi.mock("@/routes/browserSessions/BrowserSessionStream", () => ({
}));
const initialBrowserState = useStudioBrowserStore.getState();
+const initialRecordingState = useRecordingStore.getState();
// Drives a real pane-state URL write, so the effect chain under test is the
// same one a spine click goes through.
@@ -104,10 +114,29 @@ function renderStudioBrowserStream(initialPath: string) {
const BROWSER_CLOSED_PATH = "/workflows/wpid_test/studio?panes=editor";
const BROWSER_OPEN_PATH = "/workflows/wpid_test/studio?panes=editor,browser";
+const BLOCK_RUN_OPEN_PATH = `${BROWSER_OPEN_PATH}&wr=run_1&bl=Block%201`;
+const BLOCK_RUN_CLOSED_PATH = `${BROWSER_CLOSED_PATH}&wr=run_1&bl=Block%201`;
+
+function mockWorkflowRun(status: Status, browserSessionId: string | null) {
+ workflowRunQueryMock.mockReturnValue({
+ data: { status, browser_session_id: browserSessionId },
+ });
+}
+
+function controlButtonsAttr(streamButtonName: string): string | null {
+ return (
+ screen
+ .getByRole("button", { name: streamButtonName })
+ .parentElement?.getAttribute("data-show-control-buttons") ?? null
+ );
+}
beforeEach(() => {
runtimeConfigMock.browserStreamingMode = "vnc";
useStudioBrowserStore.setState(initialBrowserState, true);
+ useRecordingStore.setState(initialRecordingState, true);
+ workflowRunQueryMock.mockReset();
+ workflowRunQueryMock.mockReturnValue({ data: undefined });
});
describe("StudioBrowserStream browser activity notifications", () => {
@@ -197,3 +226,73 @@ describe("StudioBrowserStream browser activity notifications", () => {
);
});
});
+
+describe("StudioBrowserStream block-run take-control gating", () => {
+ it("locks take-control while a block run executes in the debug session", () => {
+ mockWorkflowRun(Status.Running, "pbs_test");
+ renderStudioBrowserStream(BLOCK_RUN_OPEN_PATH);
+
+ expect(controlButtonsAttr("emit vnc frame")).toBe("no");
+ expect(screen.getByRole("status").textContent).toContain(
+ "Skyvern is running this block",
+ );
+ });
+
+ it("locks the CDP stream the same way (input socket keys off the same prop)", () => {
+ runtimeConfigMock.browserStreamingMode = "cdp";
+ mockWorkflowRun(Status.Running, "pbs_test");
+ renderStudioBrowserStream(BLOCK_RUN_OPEN_PATH);
+
+ expect(controlButtonsAttr("emit cdp activity")).toBe("no");
+ expect(screen.getByRole("status").textContent).toContain(
+ "Skyvern is running this block",
+ );
+ });
+
+ it("re-enables take-control once the block run finalizes", () => {
+ mockWorkflowRun(Status.Completed, "pbs_test");
+ renderStudioBrowserStream(BLOCK_RUN_OPEN_PATH);
+
+ expect(controlButtonsAttr("emit vnc frame")).toBe("yes");
+ expect(screen.queryByRole("status")).toBeNull();
+ });
+
+ it("releases the lock while the block run is paused (may need human input)", () => {
+ mockWorkflowRun(Status.Paused, "pbs_test");
+ renderStudioBrowserStream(BLOCK_RUN_OPEN_PATH);
+
+ expect(controlButtonsAttr("emit vnc frame")).toBe("yes");
+ expect(screen.queryByRole("status")).toBeNull();
+ });
+
+ it("ignores a block run executing in a different browser session", () => {
+ mockWorkflowRun(Status.Running, "pbs_other");
+ renderStudioBrowserStream(BLOCK_RUN_OPEN_PATH);
+
+ expect(controlButtonsAttr("emit vnc frame")).toBe("yes");
+ });
+
+ it("ignores a full (non-block) run even when its session matches", () => {
+ mockWorkflowRun(Status.Running, "pbs_test");
+ renderStudioBrowserStream(`${BROWSER_OPEN_PATH}&wr=run_1`);
+
+ expect(controlButtonsAttr("emit vnc frame")).toBe("yes");
+ });
+
+ it("shows no lock pill while the Browser pane is closed", () => {
+ mockWorkflowRun(Status.Running, "pbs_test");
+ renderStudioBrowserStream(BLOCK_RUN_CLOSED_PATH);
+
+ expect(controlButtonsAttr("emit vnc frame")).toBe("no");
+ expect(screen.queryByRole("status")).toBeNull();
+ });
+
+ it("defers to an active recording instead of yanking its control", () => {
+ useRecordingStore.setState({ isRecording: true });
+ mockWorkflowRun(Status.Running, "pbs_test");
+ renderStudioBrowserStream(BLOCK_RUN_OPEN_PATH);
+
+ expect(controlButtonsAttr("emit vnc frame")).toBe("yes");
+ expect(screen.queryByRole("status")).toBeNull();
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/studio/StudioBrowserStream.tsx b/skyvern-frontend/src/routes/workflows/studio/StudioBrowserStream.tsx
index 1f0188e7e..bc3bd174a 100644
--- a/skyvern-frontend/src/routes/workflows/studio/StudioBrowserStream.tsx
+++ b/skyvern-frontend/src/routes/workflows/studio/StudioBrowserStream.tsx
@@ -6,6 +6,7 @@ import { useStudioBrowserStore } from "@/store/useStudioBrowserStore";
import { useDebugSessionQuery } from "../hooks/useDebugSessionQuery";
import { StreamPresenter } from "./StreamPresenter";
+import { useExecutingBlockRun } from "./useExecutingBlockRun";
import { useStudioPanes } from "./useStudioPanes";
/**
@@ -27,6 +28,9 @@ export function StudioBrowserStream() {
enabled: false,
});
const browserSessionId = debugSession?.browser_session_id ?? null;
+ const executingBlockRun = useExecutingBlockRun();
+ // Recording keeps control by design — the recorder is driving the browser.
+ const controlLocked = executingBlockRun && !isRecording;
useEffect(() => {
reset();
@@ -59,14 +63,31 @@ export function StudioBrowserStream() {
}
return (
- <StreamPresenter
- key={`${browserSessionId}:${reloadNonce}`}
- browserSessionId={browserSessionId}
- interactive={false}
- showControlButtons={browserPaneOpen}
- isRecording={isRecording}
- onUrlChange={handleUrlChange}
- onActivity={handleActivity}
- />
+ <div className="relative h-full w-full">
+ <StreamPresenter
+ key={`${browserSessionId}:${reloadNonce}`}
+ browserSessionId={browserSessionId}
+ interactive={false}
+ showControlButtons={browserPaneOpen && !controlLocked}
+ isRecording={isRecording}
+ onUrlChange={handleUrlChange}
+ onActivity={handleActivity}
+ />
+ {/* Explains the missing take-control; only the Browser pane offers it. */}
+ {controlLocked && browserPaneOpen ? (
+ <div
+ role="status"
+ className="pointer-events-none absolute left-1/2 top-3 z-10 flex max-w-[90%] -translate-x-1/2 items-center gap-2 rounded-md bg-black/70 px-3 py-1.5 text-xs text-white backdrop-blur duration-200 motion-safe:animate-in motion-safe:fade-in"
+ >
+ <span
+ aria-hidden
+ className="size-1.5 shrink-0 rounded-full bg-studio-accent motion-safe:animate-pulse"
+ />
+ <span className="truncate">
+ Skyvern is running this block — view only until it finishes
+ </span>
+ </div>
+ ) : null}
+ </div>
);
}
diff --git a/skyvern-frontend/src/routes/workflows/studio/useExecutingBlockRun.ts b/skyvern-frontend/src/routes/workflows/studio/useExecutingBlockRun.ts
new file mode 100644
index 000000000..4bbe7d2bd
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/studio/useExecutingBlockRun.ts
@@ -0,0 +1,39 @@
+import { useParams, useSearchParams } from "react-router-dom";
+
+import { Status } from "@/api/types";
+import { statusIsFinalized } from "@/routes/tasks/types";
+
+import { useDebugSessionQuery } from "../hooks/useDebugSessionQuery";
+import { useWorkflowRunWithWorkflowQuery } from "../hooks/useWorkflowRunWithWorkflowQuery";
+
+/**
+ * True while a block-scoped run (?wr= + ?bl=) is executing inside the current
+ * debug session: the agent owns the shared debug browser, so user take-control
+ * stays off until the run finalizes. Paused runs release the gate — a pause
+ * can be waiting on human input (2FA, verification).
+ */
+export function useExecutingBlockRun(): boolean {
+ const { workflowPermanentId } = useParams();
+ const [searchParams] = useSearchParams();
+ const isBlockRun = searchParams.has("bl");
+ const urlRunId = searchParams.get("wr");
+ const workflowRunId = isBlockRun && urlRunId ? urlRunId : undefined;
+ // Shares the run cache RunView/StudioSpine poll (5s while not finalized), so
+ // the gate releases on terminal status even with the Run pane closed.
+ const { data: workflowRun } = useWorkflowRunWithWorkflowQuery(
+ workflowRunId ? { workflowRunId } : undefined,
+ );
+ const { data: debugSession } = useDebugSessionQuery({
+ workflowPermanentId,
+ enabled: false,
+ });
+ if (!workflowRunId || !workflowRun) {
+ return false;
+ }
+ return (
+ !statusIsFinalized(workflowRun) &&
+ workflowRun.status !== Status.Paused &&
+ workflowRun.browser_session_id != null &&
+ workflowRun.browser_session_id === debugSession?.browser_session_id
+ );
+}