Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
improve time-to-render/availability-checking of BrowserStream component (#3667)
Jonathan Dobsone5e812ff671f
diff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsxindex 9d2a2f396..565c2ae30 100644--- a/skyvern-frontend/src/components/BrowserStream.tsx+++ b/skyvern-frontend/src/components/BrowserStream.tsx@@ -67,8 +67,8 @@ function BrowserStream({onClose,}: Props) {let showStream: boolean = false;- let runId: string;- let entity: "browserSession" | "task" | "workflow";+ let runId: string | null;+ let entity: "browserSession" | "task" | "workflow" | null;if (browserSessionId) {runId = browserSessionId;@@ -84,7 +84,8 @@ function BrowserStream({showStream = statusIsNotFinalized(workflow.run);entity = "workflow";} else {- throw new Error("No browser session id, task or workflow provided");+ entity = null;+ runId = null;}useQuery({@@ -120,6 +121,8 @@ function BrowserStream({const [vncDisconnectedTrigger, setVncDisconnectedTrigger] = useState(0);const prevVncConnectedRef = useRef<boolean>(false);const [isVncConnected, setIsVncConnected] = useState<boolean>(false);+ const [isCanvasReady, setIsCanvasReady] = useState<boolean>(false);+ const [isReady, setIsReady] = useState(false);const [commandDisconnectedTrigger, setCommandDisconnectedTrigger] =useState(0);const prevCommandConnectedRef = useRef<boolean>(false);@@ -131,6 +134,7 @@ function BrowserStream({setCanvasContainer(node);}, []);const rfbRef = useRef<RFB | null>(null);+ const observerRef = useRef<MutationObserver | null>(null);const clientId = useClientIdStore((state) => state.clientId);const credentialGetter = useCredentialGetter();@@ -154,6 +158,11 @@ function BrowserStream({return `${params}`;}, [clientId, credentialGetter]);+ // browser is ready+ useEffect(() => {+ setIsReady(isVncConnected && isCanvasReady && hasBrowserSession);+ }, [hasBrowserSession, isCanvasReady, isVncConnected]);+// effect for vnc disconnects onlyuseEffect(() => {if (prevVncConnectedRef.current && !isVncConnected) {@@ -218,31 +227,57 @@ function BrowserStream({throw new Error("Canvas element not found");}+ observerRef.current = new MutationObserver(() => {+ const canvasElement = canvasContainer.querySelector("canvas");+ if (canvasElement) {+ setIsCanvasReady(true);+ observerRef.current?.disconnect();+ }+ });++ observerRef.current.observe(canvasContainer, {+ childList: true,+ subtree: true,+ });+const rfb = new RFB(canvas, vncUrl);rfb.scaleViewport = true;rfbRef.current = rfb;+ const canvasElement = canvasContainer.querySelector("canvas");++ if (canvasElement) {+ setIsCanvasReady(true);+ observerRef.current?.disconnect();+ }+rfb.addEventListener("connect", () => {setIsVncConnected(true);});rfb.addEventListener("disconnect", async (/* e: RfbEvent */) => {setIsVncConnected(false);+ setIsCanvasReady(false);});- // setIsVncConnected(true); // be optimistic+ setIsVncConnected(true); // be optimistic}setupVnc();return () => {+ if (observerRef.current) {+ observerRef.current.disconnect();+ observerRef.current = null;+ }if (rfbRef.current) {rfbRef.current.disconnect();rfbRef.current = null;}setIsVncConnected(false);+ setIsCanvasReady(false);};},// cannot include isVncConnected in deps as it will cause infinite loop@@ -402,7 +437,7 @@ function BrowserStream({)}ref={setCanvasContainerRef}>- {isVncConnected && hasBrowserSession && (+ {isReady && (<div className="overlay z-10 flex items-center justify-center overflow-hidden">{showControlButtons && (<div className="control-buttons pointer-events-none relative flex h-full w-full items-center justify-center">@@ -437,7 +472,7 @@ function BrowserStream({)}</div>)}- {!isVncConnected && (+ {!isReady && (<div className="absolute left-0 top-1/2 flex aspect-video max-h-full w-full -translate-y-1/2 flex-col items-center justify-center gap-2 rounded-md border border-slate-800 text-sm text-slate-400">{browserSessionId && !hasBrowserSession ? (<div>This live browser session is no longer streaming.</div>diff --git a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsxindex d431a93aa..b5f5288bf 100644--- a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx@@ -706,16 +706,8 @@ function Workspace({version2: WorkflowVersion,mode: "visual" | "json" = "visual",) => {- console.log(- `${mode === "visual" ? "Visual" : "JSON"} comparison between versions:`,- version1.version,- "and",- version2.version,- );-// Implement visual drawer comparisonif (mode === "visual") {- console.log("Opening visual comparison panel...");// Keep history panel active but add comparison datasetWorkflowPanelState({active: true,@@ -731,15 +723,13 @@ function Workspace({// TODO: Implement JSON diff comparisonif (mode === "json") {// This will open a JSON diff view- console.log("Opening JSON diff view...");+ console.warn("[Not Implemented] opening JSON diff view...");// Future: setJsonDiffOpen(true);// Future: setJsonDiffVersions({ version1, version2 });}};const handleSelectState = (selectedVersion: WorkflowVersion) => {- console.log("Loading version into main editor:", selectedVersion.version);-// Close panelssetWorkflowPanelState({active: false,@@ -1269,51 +1259,42 @@ function Workspace({{/* browser & timeline */}<div className="flex h-[calc(100%_-_8rem)] w-full gap-6">{/* VNC browser */}- {!activeDebugSession ||- (activeDebugSession.vnc_streaming_supported && (- <div className="skyvern-vnc-browser flex h-full w-[calc(100%_-_6rem)] flex-1 flex-col items-center justify-center">- <div key={reloadKey} className="w-full flex-1">- {activeDebugSession &&- activeDebugSession.browser_session_id &&- !cycleBrowser.isPending ? (- <BrowserStream- interactive={true}- browserSessionId={- activeDebugSession.browser_session_id- }- showControlButtons={true}- resizeTrigger={windowResizeTrigger}- />- ) : (- <div className="flex aspect-video w-full flex-col items-center justify-center gap-2 rounded-md border border-slate-800 pb-2 pt-4 text-sm text-slate-400">- Connecting to your browser...- <AnimatedWave text=".‧₊˚ ⋅ ✨★ ‧₊˚ ⋅" />- </div>- )}+ {(!activeDebugSession ||+ activeDebugSession.vnc_streaming_supported) && (+ <div className="skyvern-vnc-browser flex h-full w-[calc(100%_-_6rem)] flex-1 flex-col items-center justify-center">+ <div key={reloadKey} className="w-full flex-1">+ <BrowserStream+ interactive={true}+ browserSessionId={+ activeDebugSession?.browser_session_id+ }+ showControlButtons={true}+ resizeTrigger={windowResizeTrigger}+ />+ </div>+ <footer className="flex h-[2rem] w-full items-center justify-start gap-4">+ <div className="flex items-center gap-2">+ <GlobeIcon /> Live Browser</div>- <footer className="flex h-[2rem] w-full items-center justify-start gap-4">- <div className="flex items-center gap-2">- <GlobeIcon /> Live Browser- </div>- {showBreakoutButton && (- <BreakoutButton onClick={() => breakout()} />+ {showBreakoutButton && (+ <BreakoutButton onClick={() => breakout()} />+ )}+ <div+ className={cn("ml-auto flex items-center gap-2", {+ "mr-16": !blockLabel,+ })}+ >+ {showPowerButton && (+ <PowerButton onClick={() => cycle()} />)}- <div- className={cn("ml-auto flex items-center gap-2", {- "mr-16": !blockLabel,- })}- >- {showPowerButton && (- <PowerButton onClick={() => cycle()} />- )}- <ReloadButton- isReloading={isReloading}- onClick={() => reload()}- />- </div>- </footer>- </div>- ))}+ <ReloadButton+ isReloading={isReloading}+ onClick={() => reload()}+ />+ </div>+ </footer>+ </div>+ )}{/* Screenshot browser} */}{activeDebugSession &&