Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(SKY-11665): studio Browser pane — lock take-control while a block run executes (#7008)
Celal Zamanoğlucddad3a3f005
diff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsxindex 3742c7b5f..28198468c 100644--- a/skyvern-frontend/src/components/BrowserStream.tsx+++ b/skyvern-frontend/src/components/BrowserStream.tsx@@ -722,6 +722,15 @@ function BrowserStream({}}, [interactive]);+ // When control can no longer be offered (buttons hidden and not inherently+ // interactive), a prior grab must be released or its input keeps flowing.+ // Recording is exempt: it holds take-control for exfiltration.+ useEffect(() => {+ if (!interactive && !showControlButtons && !recordingStore.isRecording) {+ setUserIsControlling(false);+ }+ }, [interactive, showControlButtons, recordingStore.isRecording]);+const theUserIsControlling =userIsControlling || (interactive && !showControlButtons);diff --git a/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsx b/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsxindex e082c1cb3..d824523bb 100644--- a/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsx+++ b/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsx@@ -152,6 +152,14 @@ function BrowserSessionStream({onActivityRef.current = onActivity;}, [onActivity]);+ // Once control can't be offered (input socket torn down), forget any prior+ // grab so re-enabling doesn't silently restore control without a new click.+ useEffect(() => {+ if (!controllable) {+ setUserIsControlling(false);+ }+ }, [controllable, setUserIsControlling]);+useEffect(() => {let cancelled = false;setStreamImgSrc("");diff --git a/skyvern-frontend/src/routes/workflows/studio/StudioBrowserStream.test.tsx b/skyvern-frontend/src/routes/workflows/studio/StudioBrowserStream.test.tsxindex d26afb05f..00e84f9ce 100644--- a/skyvern-frontend/src/routes/workflows/studio/StudioBrowserStream.test.tsx+++ b/skyvern-frontend/src/routes/workflows/studio/StudioBrowserStream.test.tsx@@ -4,6 +4,8 @@ import { fireEvent, render, screen, waitFor } from "@testing-library/react";import { MemoryRouter, Route, Routes } from "react-router-dom";import { beforeEach, describe, expect, it, vi } from "vitest";+import { Status } from "@/api/types";+import { useRecordingStore } from "@/store/useRecordingStore";import { useStudioBrowserStore } from "@/store/useStudioBrowserStore";import { StudioBrowserStream } from "./StudioBrowserStream";@@ -13,12 +15,19 @@ const runtimeConfigMock = vi.hoisted(() => ({browserStreamingMode: "vnc",}));+const workflowRunQueryMock = vi.hoisted(() => vi.fn());+vi.mock("../hooks/useDebugSessionQuery", () => ({useDebugSessionQuery: () => ({data: { browser_session_id: "pbs_test" },}),}));+vi.mock("../hooks/useWorkflowRunWithWorkflowQuery", () => ({+ useWorkflowRunWithWorkflowQuery: (options?: { workflowRunId?: string }) =>+ workflowRunQueryMock(options),+}));+vi.mock("@/hooks/useRuntimeConfig", () => ({useBrowserStreamingMode: () => ({browserStreamingMode: runtimeConfigMock.browserStreamingMode,@@ -71,6 +80,7 @@ vi.mock("@/routes/browserSessions/BrowserSessionStream", () => ({}));const initialBrowserState = useStudioBrowserStore.getState();+const initialRecordingState = useRecordingStore.getState();// Drives a real pane-state URL write, so the effect chain under test is the// same one a spine click goes through.@@ -104,10 +114,29 @@ function renderStudioBrowserStream(initialPath: string) {const BROWSER_CLOSED_PATH = "/workflows/wpid_test/studio?panes=editor";const BROWSER_OPEN_PATH = "/workflows/wpid_test/studio?panes=editor,browser";+const BLOCK_RUN_OPEN_PATH = `${BROWSER_OPEN_PATH}&wr=run_1&bl=Block%201`;+const BLOCK_RUN_CLOSED_PATH = `${BROWSER_CLOSED_PATH}&wr=run_1&bl=Block%201`;++function mockWorkflowRun(status: Status, browserSessionId: string | null) {+ workflowRunQueryMock.mockReturnValue({+ data: { status, browser_session_id: browserSessionId },+ });+}++function controlButtonsAttr(streamButtonName: string): string | null {+ return (+ screen+ .getByRole("button", { name: streamButtonName })+ .parentElement?.getAttribute("data-show-control-buttons") ?? null+ );+}beforeEach(() => {runtimeConfigMock.browserStreamingMode = "vnc";useStudioBrowserStore.setState(initialBrowserState, true);+ useRecordingStore.setState(initialRecordingState, true);+ workflowRunQueryMock.mockReset();+ workflowRunQueryMock.mockReturnValue({ data: undefined });});describe("StudioBrowserStream browser activity notifications", () => {@@ -197,3 +226,73 @@ describe("StudioBrowserStream browser activity notifications", () => {);});});++describe("StudioBrowserStream block-run take-control gating", () => {+ it("locks take-control while a block run executes in the debug session", () => {+ mockWorkflowRun(Status.Running, "pbs_test");+ renderStudioBrowserStream(BLOCK_RUN_OPEN_PATH);++ expect(controlButtonsAttr("emit vnc frame")).toBe("no");+ expect(screen.getByRole("status").textContent).toContain(+ "Skyvern is running this block",+ );+ });++ it("locks the CDP stream the same way (input socket keys off the same prop)", () => {+ runtimeConfigMock.browserStreamingMode = "cdp";+ mockWorkflowRun(Status.Running, "pbs_test");+ renderStudioBrowserStream(BLOCK_RUN_OPEN_PATH);++ expect(controlButtonsAttr("emit cdp activity")).toBe("no");+ expect(screen.getByRole("status").textContent).toContain(+ "Skyvern is running this block",+ );+ });++ it("re-enables take-control once the block run finalizes", () => {+ mockWorkflowRun(Status.Completed, "pbs_test");+ renderStudioBrowserStream(BLOCK_RUN_OPEN_PATH);++ expect(controlButtonsAttr("emit vnc frame")).toBe("yes");+ expect(screen.queryByRole("status")).toBeNull();+ });++ it("releases the lock while the block run is paused (may need human input)", () => {+ mockWorkflowRun(Status.Paused, "pbs_test");+ renderStudioBrowserStream(BLOCK_RUN_OPEN_PATH);++ expect(controlButtonsAttr("emit vnc frame")).toBe("yes");+ expect(screen.queryByRole("status")).toBeNull();+ });++ it("ignores a block run executing in a different browser session", () => {+ mockWorkflowRun(Status.Running, "pbs_other");+ renderStudioBrowserStream(BLOCK_RUN_OPEN_PATH);++ expect(controlButtonsAttr("emit vnc frame")).toBe("yes");+ });++ it("ignores a full (non-block) run even when its session matches", () => {+ mockWorkflowRun(Status.Running, "pbs_test");+ renderStudioBrowserStream(`${BROWSER_OPEN_PATH}&wr=run_1`);++ expect(controlButtonsAttr("emit vnc frame")).toBe("yes");+ });++ it("shows no lock pill while the Browser pane is closed", () => {+ mockWorkflowRun(Status.Running, "pbs_test");+ renderStudioBrowserStream(BLOCK_RUN_CLOSED_PATH);++ expect(controlButtonsAttr("emit vnc frame")).toBe("no");+ expect(screen.queryByRole("status")).toBeNull();+ });++ it("defers to an active recording instead of yanking its control", () => {+ useRecordingStore.setState({ isRecording: true });+ mockWorkflowRun(Status.Running, "pbs_test");+ renderStudioBrowserStream(BLOCK_RUN_OPEN_PATH);++ expect(controlButtonsAttr("emit vnc frame")).toBe("yes");+ expect(screen.queryByRole("status")).toBeNull();+ });+});diff --git a/skyvern-frontend/src/routes/workflows/studio/StudioBrowserStream.tsx b/skyvern-frontend/src/routes/workflows/studio/StudioBrowserStream.tsxindex 1f0188e7e..bc3bd174a 100644--- a/skyvern-frontend/src/routes/workflows/studio/StudioBrowserStream.tsx+++ b/skyvern-frontend/src/routes/workflows/studio/StudioBrowserStream.tsx@@ -6,6 +6,7 @@ import { useStudioBrowserStore } from "@/store/useStudioBrowserStore";import { useDebugSessionQuery } from "../hooks/useDebugSessionQuery";import { StreamPresenter } from "./StreamPresenter";+import { useExecutingBlockRun } from "./useExecutingBlockRun";import { useStudioPanes } from "./useStudioPanes";/**@@ -27,6 +28,9 @@ export function StudioBrowserStream() {enabled: false,});const browserSessionId = debugSession?.browser_session_id ?? null;+ const executingBlockRun = useExecutingBlockRun();+ // Recording keeps control by design — the recorder is driving the browser.+ const controlLocked = executingBlockRun && !isRecording;useEffect(() => {reset();@@ -59,14 +63,31 @@ export function StudioBrowserStream() {}return (- <StreamPresenter- key={`${browserSessionId}:${reloadNonce}`}- browserSessionId={browserSessionId}- interactive={false}- showControlButtons={browserPaneOpen}- isRecording={isRecording}- onUrlChange={handleUrlChange}- onActivity={handleActivity}- />+ <div className="relative h-full w-full">+ <StreamPresenter+ key={`${browserSessionId}:${reloadNonce}`}+ browserSessionId={browserSessionId}+ interactive={false}+ showControlButtons={browserPaneOpen && !controlLocked}+ isRecording={isRecording}+ onUrlChange={handleUrlChange}+ onActivity={handleActivity}+ />+ {/* Explains the missing take-control; only the Browser pane offers it. */}+ {controlLocked && browserPaneOpen ? (+ <div+ role="status"+ className="pointer-events-none absolute left-1/2 top-3 z-10 flex max-w-[90%] -translate-x-1/2 items-center gap-2 rounded-md bg-black/70 px-3 py-1.5 text-xs text-white backdrop-blur duration-200 motion-safe:animate-in motion-safe:fade-in"+ >+ <span+ aria-hidden+ className="size-1.5 shrink-0 rounded-full bg-studio-accent motion-safe:animate-pulse"+ />+ <span className="truncate">+ Skyvern is running this block — view only until it finishes+ </span>+ </div>+ ) : null}+ </div>);}diff --git a/skyvern-frontend/src/routes/workflows/studio/useExecutingBlockRun.ts b/skyvern-frontend/src/routes/workflows/studio/useExecutingBlockRun.tsnew file mode 100644index 000000000..4bbe7d2bd--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/studio/useExecutingBlockRun.ts@@ -0,0 +1,39 @@+import { useParams, useSearchParams } from "react-router-dom";++import { Status } from "@/api/types";+import { statusIsFinalized } from "@/routes/tasks/types";++import { useDebugSessionQuery } from "../hooks/useDebugSessionQuery";+import { useWorkflowRunWithWorkflowQuery } from "../hooks/useWorkflowRunWithWorkflowQuery";++/**+ * True while a block-scoped run (?wr= + ?bl=) is executing inside the current+ * debug session: the agent owns the shared debug browser, so user take-control+ * stays off until the run finalizes. Paused runs release the gate — a pause+ * can be waiting on human input (2FA, verification).+ */+export function useExecutingBlockRun(): boolean {+ const { workflowPermanentId } = useParams();+ const [searchParams] = useSearchParams();+ const isBlockRun = searchParams.has("bl");+ const urlRunId = searchParams.get("wr");+ const workflowRunId = isBlockRun && urlRunId ? urlRunId : undefined;+ // Shares the run cache RunView/StudioSpine poll (5s while not finalized), so+ // the gate releases on terminal status even with the Run pane closed.+ const { data: workflowRun } = useWorkflowRunWithWorkflowQuery(+ workflowRunId ? { workflowRunId } : undefined,+ );+ const { data: debugSession } = useDebugSessionQuery({+ workflowPermanentId,+ enabled: false,+ });+ if (!workflowRunId || !workflowRun) {+ return false;+ }+ return (+ !statusIsFinalized(workflowRun) &&+ workflowRun.status !== Status.Paused &&+ workflowRun.browser_session_id != null &&+ workflowRun.browser_session_id === debugSession?.browser_session_id+ );+}