Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Remove frontend hack for requesting persistent browser sessions, part iii (frontend) (#3053)
Jonathan Dobson9c00cff36243
diff --git a/skyvern-frontend/src/api/types.ts b/skyvern-frontend/src/api/types.tsindex fbc90ca59..dea2c6716 100644--- a/skyvern-frontend/src/api/types.ts+++ b/skyvern-frontend/src/api/types.ts@@ -266,6 +266,14 @@ export interface EvalTask extends Eval {export type EvalApiResponse = EvalWorkflow[] | EvalTask[];+export type DebugSessionApiResponse = {+ debug_session_id: string;+ browser_session_id: string;+ workflow_permanent_id: string | null;+ created_at: string;+ modified_at: string;+};+export type WorkflowRunApiResponse = {created_at: string;failure_reason: string | null;diff --git a/skyvern-frontend/src/routes/workflows/editor/WorkflowDebugger.tsx b/skyvern-frontend/src/routes/workflows/editor/WorkflowDebugger.tsxindex 5886ed00c..ab0bfe62a 100644--- a/skyvern-frontend/src/routes/workflows/editor/WorkflowDebugger.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/WorkflowDebugger.tsx@@ -1,5 +1,7 @@import { ReactFlowProvider } from "@xyflow/react";import { useParams } from "react-router-dom";+import { useQueryClient } from "@tanstack/react-query";+import { useEffect, useRef, useState } from "react";import { BrowserStream } from "@/components/BrowserStream";import { FloatingWindow } from "@/components/FloatingWindow";@@ -10,6 +12,7 @@ import { useSidebarStore } from "@/store/SidebarStore";import { useWorkflowHasChangesStore } from "@/store/WorkflowHasChangesStore";import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";import { useWorkflowQuery } from "../hooks/useWorkflowQuery";+import { useDebugSessionQuery } from "../hooks/useDebugSessionQuery";import { WorkflowSettings } from "../types/workflowTypes";import { FlowRenderer } from "./FlowRenderer";import { getElements } from "./workflowEditorUtils";@@ -17,12 +20,19 @@ import { getInitialParameters } from "./utils";function WorkflowDebugger() {const { workflowPermanentId } = useParams();+ const queryClient = useQueryClient();+ const [shouldFetchDebugSession, setShouldFetchDebugSession] = useState(false);const { data: workflowRun } = useWorkflowRunQuery();const { data: workflow } = useWorkflowQuery({workflowPermanentId,});+ const { data: debugSession } = useDebugSessionQuery({+ workflowPermanentId,+ enabled: shouldFetchDebugSession && !!workflowPermanentId,+ });+const setCollapsed = useSidebarStore((state) => {return state.setCollapsed;});@@ -34,8 +44,42 @@ function WorkflowDebugger() {useMountEffect(() => {setCollapsed(true);setHasChanges(false);++ if (workflowPermanentId) {+ queryClient.removeQueries({+ queryKey: ["debugSession", workflowPermanentId],+ });+ setShouldFetchDebugSession(true);+ }});+ const intervalRef = useRef<NodeJS.Timeout | null>(null);++ useEffect(() => {+ if (+ (!debugSession || !debugSession.browser_session_id) &&+ shouldFetchDebugSession &&+ workflowPermanentId+ ) {+ intervalRef.current = setInterval(() => {+ queryClient.invalidateQueries({+ queryKey: ["debugSession", workflowPermanentId],+ });+ }, 2000);+ } else {+ if (intervalRef.current) {+ clearInterval(intervalRef.current);+ intervalRef.current = null;+ }+ }++ return () => {+ if (intervalRef.current) {+ clearInterval(intervalRef.current);+ }+ };+ }, [debugSession, shouldFetchDebugSession, workflowPermanentId, queryClient]);+if (!workflow) {return null;}@@ -73,7 +117,7 @@ function WorkflowDebugger() {/></ReactFlowProvider>- {workflowRun && (+ {debugSession && (<FloatingWindowtitle={browserTitle}bounded={false}@@ -83,10 +127,10 @@ function WorkflowDebugger() {showMinimizeButton={true}showReloadButton={true}>- {workflowRun && workflowRun.browser_session_id ? (+ {debugSession && debugSession.browser_session_id ? (<BrowserStreaminteractive={interactor === "human"}- browserSessionId={workflowRun.browser_session_id}+ browserSessionId={debugSession.browser_session_id}/>) : (<Skeleton className="h-full w-full" />diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsxindex 4c2a33023..16030f791 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx@@ -5,12 +5,14 @@ import { useLocation, useNavigate, useParams } from "react-router-dom";import { useMutation, useQueryClient } from "@tanstack/react-query";import { getClient } from "@/api/AxiosClient";-import { ProxyLocation, User } from "@/api/types";+import { ProxyLocation } from "@/api/types";import { Timer } from "@/components/Timer";import { toast } from "@/components/ui/use-toast";import { useCredentialGetter } from "@/hooks/useCredentialGetter";+import { useNodeLabelChangeHandler } from "@/routes/workflows/hooks/useLabelChangeHandler";import { useDeleteNodeCallback } from "@/routes/workflows/hooks/useDeleteNodeCallback";+import { useDebugSessionQuery } from "@/routes/workflows/hooks/useDebugSessionQuery";import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";import {debuggableWorkflowBlockTypes,@@ -29,11 +31,6 @@ import {statusIsFinalized,statusIsRunningOrQueued,} from "@/routes/tasks/types";-import {- useOptimisticallyRequestBrowserSessionId,- type OptimisticBrowserSession,-} from "@/store/useOptimisticallyRequestBrowserSessionId";-import { useUser } from "@/hooks/useUser";import { EditableNodeTitle } from "../components/EditableNodeTitle";import { NodeActionMenu } from "../NodeActionMenu";@@ -73,24 +70,13 @@ const blockTypeToTitle = (type: WorkflowBlockType): string => {const getPayload = (opts: {blockLabel: string;- optimistic: OptimisticBrowserSession;+ browserSessionId: string | null;parameters: Record<string, unknown>;totpIdentifier: string | null;totpUrl: string | null;- user: User | null;workflowPermanentId: string;workflowSettings: WorkflowSettingsState;}): Payload | null => {- if (!opts.user) {- toast({- variant: "warning",- title: "Error",- description: "No user found",- });-- return null;- }-const webhook_url = opts.workflowSettings.webhookCallbackUrl.trim();let extraHttpHeaders = null;@@ -108,14 +94,7 @@ const getPayload = (opts: {});}- const browserSessionData = opts.optimistic.get(- opts.user,- opts.workflowPermanentId,- );-- const browserSessionId = browserSessionData?.browser_session_id;-- if (!browserSessionId) {+ if (!opts.browserSessionId) {toast({variant: "warning",title: "Error",@@ -127,13 +106,13 @@ const getPayload = (opts: {toast({variant: "default",title: "Success",- description: `Browser session ID found: ${browserSessionId}`,+ description: `Browser session ID found: ${opts.browserSessionId}`,});}const payload: Payload = {block_labels: [opts.blockLabel],- browser_session_id: browserSessionId,+ browser_session_id: opts.browserSessionId,extra_http_headers: extraHttpHeaders,max_screenshot_scrolls: opts.workflowSettings.maxScreenshotScrollingTimes,parameters: opts.parameters,@@ -181,8 +160,9 @@ function NodeHeader({const { data: workflowRun } = useWorkflowRunQuery();const workflowRunIsRunningOrQueued =workflowRun && statusIsRunningOrQueued(workflowRun);- const optimistic = useOptimisticallyRequestBrowserSessionId();- const user = useUser().get();+ const { data: debugSession } = useDebugSessionQuery({+ workflowPermanentId,+ });useEffect(() => {if (!workflowRun || !workflowPermanentId || !workflowRunId) {@@ -220,6 +200,21 @@ function NodeHeader({mutationFn: async () => {if (!workflowPermanentId) {console.error("There is no workflowPermanentId");+ toast({+ variant: "destructive",+ title: "Failed to start workflow block run",+ description: "There is no workflowPermanentId",+ });+ return;+ }++ if (!debugSession) {+ console.error("There is no debug session, yet");+ toast({+ variant: "destructive",+ title: "Failed to start workflow block run",+ description: "There is no debug session, yet",+ });return;}@@ -244,16 +239,20 @@ function NodeHeader({const body = getPayload({blockLabel,- optimistic,+ browserSessionId: debugSession.browser_session_id,parameters,totpIdentifier,totpUrl,- user,workflowPermanentId,workflowSettings: workflowSettingsStore,});if (!body) {+ toast({+ variant: "destructive",+ title: "Failed to start workflow block run",+ description: "Could not construct run payload",+ });return;}@@ -265,6 +264,11 @@ function NodeHeader({onSuccess: (response) => {if (!response) {console.error("No response");+ toast({+ variant: "destructive",+ title: "Failed to start workflow block run",+ description: "No response",+ });return;}@@ -290,11 +294,17 @@ function NodeHeader({const cancelBlock = useMutation({mutationFn: async () => {- const browserSessionId =- user && workflowPermanentId- ? optimistic.get(user, workflowPermanentId)?.browser_session_id ??- "<missing-browser-session-id>"- : "<missing-user-or-workflow-permanent-id>";+ if (!debugSession) {+ console.error("Missing debug session");+ toast({+ variant: "destructive",+ title: "Failed to cancel workflow block run",+ description: "Missing debug session",+ });+ return;+ }++ const browserSessionId = debugSession.browser_session_id;const client = await getClient(credentialGetter);return client.post(`/runs/${browserSessionId}/workflow_run/${workflowRunId}/cancel/`)diff --git a/skyvern-frontend/src/routes/workflows/hooks/useDebugSessionQuery.ts b/skyvern-frontend/src/routes/workflows/hooks/useDebugSessionQuery.tsnew file mode 100644index 000000000..6d59df115--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/hooks/useDebugSessionQuery.ts@@ -0,0 +1,29 @@+import { getClient } from "@/api/AxiosClient";+import { DebugSessionApiResponse } from "@/api/types";+import { useCredentialGetter } from "@/hooks/useCredentialGetter";+import { useQuery } from "@tanstack/react-query";++interface Opts {+ workflowPermanentId?: string;+ enabled?: boolean;+}++function useDebugSessionQuery({ workflowPermanentId, enabled }: Opts) {+ const credentialGetter = useCredentialGetter();++ return useQuery<DebugSessionApiResponse>({+ queryKey: ["debugSession", workflowPermanentId],+ queryFn: async () => {+ const client = await getClient(credentialGetter, "sans-api-v1");+ return client+ .get(`/debug-session/${workflowPermanentId}`)+ .then((response) => response.data);+ },+ enabled:+ enabled !== undefined+ ? enabled && !!workflowPermanentId+ : !!workflowPermanentId,+ });+}++export { useDebugSessionQuery };diff --git a/skyvern-frontend/src/store/useOptimisticallyRequestBrowserSessionId.ts b/skyvern-frontend/src/store/useOptimisticallyRequestBrowserSessionId.tsdeleted file mode 100644index b2e24bfbe..000000000--- a/skyvern-frontend/src/store/useOptimisticallyRequestBrowserSessionId.ts+++ /dev/null@@ -1,128 +0,0 @@-import { AxiosInstance } from "axios";-import { create as createStore } from "zustand";--import { User } from "@/api/types";-import { lsKeys } from "@/util/env";--export interface BrowserSessionData {- browser_session_id: string | null;- expires_at: number | null; // seconds since epoch-}--interface RunOpts {- client: AxiosInstance;- reason?: string;- user: User;- workflowPermanentId?: string;-}--export interface OptimisticBrowserSession {- get: (user: User, workflowPermanentId: string) => BrowserSessionData | null;- run: (runOpts: RunOpts) => Promise<BrowserSessionData>;-}--const SESSION_TIMEOUT_MINUTES = 60 * 4;-const SPARE = "spare";--const makeKey = (user: User, workflowPermanentId?: string | undefined) => {- return `${lsKeys.optimisticBrowserSession}:${user.id}:${workflowPermanentId ?? SPARE}`;-};--/**- * Read a `BrowserSessionData` from localStorage cache. If the entry is expired,- * return `null`. If the entry is invalid, return `null`. Otherwise return it.- */-const read = (key: string): BrowserSessionData | null => {- const stored = localStorage.getItem(key);- if (stored) {- try {- const parsed = JSON.parse(stored);- const { browser_session_id, expires_at } = parsed;- const now = Math.floor(Date.now() / 1000); // seconds since epoch-- if (- browser_session_id &&- typeof browser_session_id === "string" &&- expires_at &&- typeof expires_at === "number" &&- now < expires_at- ) {- return { browser_session_id, expires_at };- }- } catch (e) {- // pass- }- }-- return null;-};--/**- * Write a `BrowserSessionData` to localStorage cache.- */-const write = (key: string, browserSessionData: BrowserSessionData) => {- localStorage.setItem(key, JSON.stringify(browserSessionData));-};--/**- * Delete a localStorage key.- */-const del = (key: string) => {- localStorage.removeItem(key);-};--/**- * Create a new browser session and return the `BrowserSessionData`.- */-const create = async (client: AxiosInstance): Promise<BrowserSessionData> => {- const resp = await client.post("/browser_sessions", {- timeout: SESSION_TIMEOUT_MINUTES,- });-- const { browser_session_id: newBrowserSessionId, timeout } = resp.data;- const newExpiresAt = Math.floor(Date.now() / 1000) + timeout * 60 * 0.9;-- return {- browser_session_id: newBrowserSessionId,- expires_at: newExpiresAt,- };-};--export const useOptimisticallyRequestBrowserSessionId =- createStore<OptimisticBrowserSession>(() => ({- get: (user: User, workflowPermanentId: string) => {- return read(makeKey(user, workflowPermanentId));- },- run: async ({ client, user, workflowPermanentId }: RunOpts) => {- if (workflowPermanentId) {- const userKey = makeKey(user, workflowPermanentId);- const exists = read(userKey);-- if (exists) {- return exists;- }-- const spareKey = makeKey(user, SPARE);- const spare = read(spareKey);-- if (spare) {- del(spareKey);- write(userKey, spare);- create(client).then((newSpare) => write(spareKey, newSpare));- return spare;- }- }-- const key = makeKey(user, workflowPermanentId);- const browserSessionData = read(key);-- if (browserSessionData) {- return browserSessionData;- }-- const knew = await create(client);- write(key, knew);-- return knew;- },- }));diff --git a/skyvern-frontend/src/util/env.ts b/skyvern-frontend/src/util/env.tsindex faba1b04e..99b4f0902 100644--- a/skyvern-frontend/src/util/env.ts+++ b/skyvern-frontend/src/util/env.ts@@ -23,7 +23,6 @@ const apiPathPrefix = import.meta.env.VITE_API_PATH_PREFIX ?? "";const lsKeys = {browserSessionId: "skyvern.browserSessionId",- optimisticBrowserSession: "skyvern.optimisticBrowserSession",};const wssBaseUrl = import.meta.env.VITE_WSS_BASE_URL;