Skip to content

Automate browser based workflows with AI

improve time-to-render/availability-checking of BrowserStream component (#3667)

Jonathan Dobsone5e812ff671f
diff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsx
index 9d2a2f396..565c2ae30 100644
--- a/skyvern-frontend/src/components/BrowserStream.tsx
+++ b/skyvern-frontend/src/components/BrowserStream.tsx
@@ -67,8 +67,8 @@ function BrowserStream({
onClose,
}: Props) {
let showStream: boolean = false;
- let runId: string;
- let entity: "browserSession" | "task" | "workflow";
+ let runId: string | null;
+ let entity: "browserSession" | "task" | "workflow" | null;
if (browserSessionId) {
runId = browserSessionId;
@@ -84,7 +84,8 @@ function BrowserStream({
showStream = statusIsNotFinalized(workflow.run);
entity = "workflow";
} else {
- throw new Error("No browser session id, task or workflow provided");
+ entity = null;
+ runId = null;
}
useQuery({
@@ -120,6 +121,8 @@ function BrowserStream({
const [vncDisconnectedTrigger, setVncDisconnectedTrigger] = useState(0);
const prevVncConnectedRef = useRef<boolean>(false);
const [isVncConnected, setIsVncConnected] = useState<boolean>(false);
+ const [isCanvasReady, setIsCanvasReady] = useState<boolean>(false);
+ const [isReady, setIsReady] = useState(false);
const [commandDisconnectedTrigger, setCommandDisconnectedTrigger] =
useState(0);
const prevCommandConnectedRef = useRef<boolean>(false);
@@ -131,6 +134,7 @@ function BrowserStream({
setCanvasContainer(node);
}, []);
const rfbRef = useRef<RFB | null>(null);
+ const observerRef = useRef<MutationObserver | null>(null);
const clientId = useClientIdStore((state) => state.clientId);
const credentialGetter = useCredentialGetter();
@@ -154,6 +158,11 @@ function BrowserStream({
return `${params}`;
}, [clientId, credentialGetter]);
+ // browser is ready
+ useEffect(() => {
+ setIsReady(isVncConnected && isCanvasReady && hasBrowserSession);
+ }, [hasBrowserSession, isCanvasReady, isVncConnected]);
+
// effect for vnc disconnects only
useEffect(() => {
if (prevVncConnectedRef.current && !isVncConnected) {
@@ -218,31 +227,57 @@ function BrowserStream({
throw new Error("Canvas element not found");
}
+ observerRef.current = new MutationObserver(() => {
+ const canvasElement = canvasContainer.querySelector("canvas");
+ if (canvasElement) {
+ setIsCanvasReady(true);
+ observerRef.current?.disconnect();
+ }
+ });
+
+ observerRef.current.observe(canvasContainer, {
+ childList: true,
+ subtree: true,
+ });
+
const rfb = new RFB(canvas, vncUrl);
rfb.scaleViewport = true;
rfbRef.current = rfb;
+ const canvasElement = canvasContainer.querySelector("canvas");
+
+ if (canvasElement) {
+ setIsCanvasReady(true);
+ observerRef.current?.disconnect();
+ }
+
rfb.addEventListener("connect", () => {
setIsVncConnected(true);
});
rfb.addEventListener("disconnect", async (/* e: RfbEvent */) => {
setIsVncConnected(false);
+ setIsCanvasReady(false);
});
- // setIsVncConnected(true); // be optimistic
+ setIsVncConnected(true); // be optimistic
}
setupVnc();
return () => {
+ if (observerRef.current) {
+ observerRef.current.disconnect();
+ observerRef.current = null;
+ }
if (rfbRef.current) {
rfbRef.current.disconnect();
rfbRef.current = null;
}
setIsVncConnected(false);
+ setIsCanvasReady(false);
};
},
// cannot include isVncConnected in deps as it will cause infinite loop
@@ -402,7 +437,7 @@ function BrowserStream({
)}
ref={setCanvasContainerRef}
>
- {isVncConnected && hasBrowserSession && (
+ {isReady && (
<div className="overlay z-10 flex items-center justify-center overflow-hidden">
{showControlButtons && (
<div className="control-buttons pointer-events-none relative flex h-full w-full items-center justify-center">
@@ -437,7 +472,7 @@ function BrowserStream({
)}
</div>
)}
- {!isVncConnected && (
+ {!isReady && (
<div className="absolute left-0 top-1/2 flex aspect-video max-h-full w-full -translate-y-1/2 flex-col items-center justify-center gap-2 rounded-md border border-slate-800 text-sm text-slate-400">
{browserSessionId && !hasBrowserSession ? (
<div>This live browser session is no longer streaming.</div>
diff --git a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
index d431a93aa..b5f5288bf 100644
--- a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
@@ -706,16 +706,8 @@ function Workspace({
version2: WorkflowVersion,
mode: "visual" | "json" = "visual",
) => {
- console.log(
- `${mode === "visual" ? "Visual" : "JSON"} comparison between versions:`,
- version1.version,
- "and",
- version2.version,
- );
-
// Implement visual drawer comparison
if (mode === "visual") {
- console.log("Opening visual comparison panel...");
// Keep history panel active but add comparison data
setWorkflowPanelState({
active: true,
@@ -731,15 +723,13 @@ function Workspace({
// TODO: Implement JSON diff comparison
if (mode === "json") {
// This will open a JSON diff view
- console.log("Opening JSON diff view...");
+ console.warn("[Not Implemented] opening JSON diff view...");
// Future: setJsonDiffOpen(true);
// Future: setJsonDiffVersions({ version1, version2 });
}
};
const handleSelectState = (selectedVersion: WorkflowVersion) => {
- console.log("Loading version into main editor:", selectedVersion.version);
-
// Close panels
setWorkflowPanelState({
active: false,
@@ -1269,51 +1259,42 @@ function Workspace({
{/* browser & timeline */}
<div className="flex h-[calc(100%_-_8rem)] w-full gap-6">
{/* VNC browser */}
- {!activeDebugSession ||
- (activeDebugSession.vnc_streaming_supported && (
- <div className="skyvern-vnc-browser flex h-full w-[calc(100%_-_6rem)] flex-1 flex-col items-center justify-center">
- <div key={reloadKey} className="w-full flex-1">
- {activeDebugSession &&
- activeDebugSession.browser_session_id &&
- !cycleBrowser.isPending ? (
- <BrowserStream
- interactive={true}
- browserSessionId={
- activeDebugSession.browser_session_id
- }
- showControlButtons={true}
- resizeTrigger={windowResizeTrigger}
- />
- ) : (
- <div className="flex aspect-video w-full flex-col items-center justify-center gap-2 rounded-md border border-slate-800 pb-2 pt-4 text-sm text-slate-400">
- Connecting to your browser...
- <AnimatedWave text=".‧₊˚ ⋅ ✨★ ‧₊˚ ⋅" />
- </div>
- )}
+ {(!activeDebugSession ||
+ activeDebugSession.vnc_streaming_supported) && (
+ <div className="skyvern-vnc-browser flex h-full w-[calc(100%_-_6rem)] flex-1 flex-col items-center justify-center">
+ <div key={reloadKey} className="w-full flex-1">
+ <BrowserStream
+ interactive={true}
+ browserSessionId={
+ activeDebugSession?.browser_session_id
+ }
+ showControlButtons={true}
+ resizeTrigger={windowResizeTrigger}
+ />
+ </div>
+ <footer className="flex h-[2rem] w-full items-center justify-start gap-4">
+ <div className="flex items-center gap-2">
+ <GlobeIcon /> Live Browser
</div>
- <footer className="flex h-[2rem] w-full items-center justify-start gap-4">
- <div className="flex items-center gap-2">
- <GlobeIcon /> Live Browser
- </div>
- {showBreakoutButton && (
- <BreakoutButton onClick={() => breakout()} />
+ {showBreakoutButton && (
+ <BreakoutButton onClick={() => breakout()} />
+ )}
+ <div
+ className={cn("ml-auto flex items-center gap-2", {
+ "mr-16": !blockLabel,
+ })}
+ >
+ {showPowerButton && (
+ <PowerButton onClick={() => cycle()} />
)}
- <div
- className={cn("ml-auto flex items-center gap-2", {
- "mr-16": !blockLabel,
- })}
- >
- {showPowerButton && (
- <PowerButton onClick={() => cycle()} />
- )}
- <ReloadButton
- isReloading={isReloading}
- onClick={() => reload()}
- />
- </div>
- </footer>
- </div>
- ))}
+ <ReloadButton
+ isReloading={isReloading}
+ onClick={() => reload()}
+ />
+ </div>
+ </footer>
+ </div>
+ )}
{/* Screenshot browser} */}
{activeDebugSession &&