Skip to content

Automate browser based workflows with AI

fix(SKY-11367): enable take-control and centering for studio CDP browser (#6810)

Celal Zamanoğlu8deb5d5dc0df
diff --git a/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsx b/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsx
index 6a066c992..a7ced06a1 100644
--- a/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsx
+++ b/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsx
@@ -95,6 +95,7 @@ interface Props {
browserSessionId: string;
interactive?: boolean;
showControlButtons?: boolean;
+ centered?: boolean;
onReadyChange?: (isReady: boolean, browserSessionId: string | null) => void;
onUrlChange?: (url: string) => void;
}
@@ -103,6 +104,7 @@ function BrowserSessionStream({
browserSessionId,
interactive = false,
showControlButtons = false,
+ centered = false,
onReadyChange,
onUrlChange,
}: Props) {
@@ -122,7 +124,11 @@ function BrowserSessionStream({
const terminalStatusSeenRef = useRef(false);
const reconnectTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
- const inputWsUrl = interactive
+ // The CDP input socket must be wired whenever the stream can be controlled,
+ // whether by default interaction or via the take-control button.
+ const controllable = interactive || showControlButtons;
+
+ const inputWsUrl = controllable
? `${newWssBaseUrl}/stream/cdp_input/browser_session/${browserSessionId}`
: null;
@@ -134,7 +140,7 @@ function BrowserSessionStream({
handlers,
} = useCdpInput({
inputWsUrl,
- interactive,
+ interactive: controllable,
viewportWidth,
viewportHeight,
});
@@ -310,7 +316,7 @@ function BrowserSessionStream({
<InteractiveStreamView
streamImgSrc={streamImgSrc}
streamFormat={streamFormat}
- interactive={interactive}
+ interactive={controllable}
userIsControlling={userIsControlling}
setUserIsControlling={setUserIsControlling}
inputReady={inputReady}
@@ -318,6 +324,7 @@ function BrowserSessionStream({
showControlButtons={showControlButtons}
handlers={handlers}
currentUrl={currentUrl}
+ centered={centered}
/>
);
}
diff --git a/skyvern-frontend/src/routes/streaming/InteractiveStreamView.tsx b/skyvern-frontend/src/routes/streaming/InteractiveStreamView.tsx
index 426ca3a77..6f98a149f 100644
--- a/skyvern-frontend/src/routes/streaming/InteractiveStreamView.tsx
+++ b/skyvern-frontend/src/routes/streaming/InteractiveStreamView.tsx
@@ -1,5 +1,5 @@
import type { RefObject } from "react";
-import { GlobeIcon } from "@radix-ui/react-icons";
+import { ExitIcon, GlobeIcon, HandIcon } from "@radix-ui/react-icons";
import { ZoomableImage } from "@/components/ZoomableImage";
import { Button } from "@/components/ui/button";
import { cn } from "@/util/utils";
@@ -60,16 +60,23 @@ function InteractiveStreamView({
{currentUrl && <UrlBar url={currentUrl} />}
{showControlButtons && !userIsControlling && inputReady && (
<div className="absolute inset-0 z-10 flex items-center justify-center">
- <Button onClick={() => setUserIsControlling(true)}>
+ <Button
+ size="sm"
+ className="border"
+ onClick={() => setUserIsControlling(true)}
+ >
+ <HandIcon className="mr-2 h-4 w-4" />
take control
</Button>
</div>
)}
{showControlButtons && userIsControlling && (
<Button
- className="absolute bottom-2 left-1/2 z-10 -translate-x-1/2"
+ size="sm"
+ className="absolute bottom-2 left-1/2 z-10 -translate-x-1/2 border"
onClick={() => setUserIsControlling(false)}
>
+ <ExitIcon className="mr-2 h-4 w-4" />
stop controlling
</Button>
)}
diff --git a/skyvern-frontend/src/routes/workflows/studio/StreamPresenter.tsx b/skyvern-frontend/src/routes/workflows/studio/StreamPresenter.tsx
index 7657b44c0..fc5dc0f4b 100644
--- a/skyvern-frontend/src/routes/workflows/studio/StreamPresenter.tsx
+++ b/skyvern-frontend/src/routes/workflows/studio/StreamPresenter.tsx
@@ -27,12 +27,14 @@ export function StreamPresenter({
const useCdp = browserStreamingMode === "cdp" && !isRecording;
if (useCdp) {
+ // CDP frames must be explicitly centered; VNC handles this in its own CSS.
return (
<BrowserSessionStream
browserSessionId={browserSessionId}
interactive={interactive}
showControlButtons={showControlButtons}
onUrlChange={onUrlChange}
+ centered
/>
);
}