Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Pre-convo UI (#3376)
Jonathan Dobson50e0597c84a4
diff --git a/skyvern-frontend/src/components/Orgwalled.tsx b/skyvern-frontend/src/components/Orgwalled.tsxindex 532492457..372515918 100644--- a/skyvern-frontend/src/components/Orgwalled.tsx+++ b/skyvern-frontend/src/components/Orgwalled.tsx@@ -11,9 +11,11 @@ import { cn } from "@/util/utils";function OrgWalled({children,className,+ hideTooltipContent,}: {children: React.ReactNode;className?: string;+ hideTooltipContent?: boolean;}) {const isSkyvernUser = useIsSkyvernUser();@@ -35,9 +37,13 @@ function OrgWalled({{children}</div></TooltipTrigger>- <TooltipContent>- <p>This feature is only available to Skyvern organization members</p>- </TooltipContent>+ {!hideTooltipContent && (+ <TooltipContent>+ <p>+ This feature is only available to Skyvern organization members+ </p>+ </TooltipContent>+ )}</Tooltip></TooltipProvider>);diff --git a/skyvern-frontend/src/components/icons/BugIcon.tsx b/skyvern-frontend/src/components/icons/BugIcon.tsxnew file mode 100644index 000000000..f6f12846a--- /dev/null+++ b/skyvern-frontend/src/components/icons/BugIcon.tsx@@ -0,0 +1,25 @@+type Props = {+ className?: string;+};++function BugIcon({ className }: Props) {+ return (+ <svg+ xmlns="http://www.w3.org/2000/svg"+ width="24"+ height="24"+ viewBox="0 0 24 24"+ fill="none"+ className={className}+ >+ <path+ clipRule="evenodd"+ fill="currentColor"+ fillRule="evenodd"+ d="M7.293 3.293a1 1 0 0 1 1.414 0l1.876 1.876A6.39 6.39 0 0 1 12 5c.515 0 .996.049 1.445.14l1.848-1.847a1 1 0 1 1 1.414 1.414L15.45 5.965A5.5 5.5 0 0 1 17.249 8H18c.173 0 .456-.06.666-.212.159-.114.334-.314.334-.788a1 1 0 1 1 2 0c0 1.126-.491 1.926-1.166 2.412A3.233 3.233 0 0 1 18 10h-.086c.06.36.086.7.086 1v1h2a1 1 0 1 1 0 2h-2v1c0 .3-.026.64-.086 1H18c.493 0 1.211.14 1.834.588C20.51 17.075 21 17.875 21 19a1 1 0 1 1-2 0c0-.474-.175-.674-.334-.788A1.239 1.239 0 0 0 18 18h-.751a5.537 5.537 0 0 1-1.552 1.857C14.766 20.563 13.543 21 12 21c-1.543 0-2.765-.437-3.697-1.143-.7-.53-1.2-1.188-1.552-1.857H6c-.173 0-.456.06-.666.212-.159.114-.334.314-.334.788a1 1 0 1 1-2 0c0-1.126.492-1.926 1.166-2.412A3.233 3.233 0 0 1 6 16h.086c-.06-.36-.086-.7-.086-1v-1H4a1 1 0 1 1 0-2h2v-1c0-.349.022-.682.065-1H6c-.493 0-1.211-.14-1.834-.588C3.492 8.926 3 8.126 3 7a1 1 0 0 1 2 0c0 .474.175.674.334.788.21.152.493.212.666.212h.696A5.34 5.34 0 0 1 8.58 5.994L7.293 4.707a1 1 0 0 1 0-1.414zM12 9a1 1 0 1 0 0 2h.001a1 1 0 1 0 0-2H12zm-3 4a1 1 0 0 1 1-1h.001a1 1 0 1 1 0 2H10a1 1 0 0 1-1-1zm5-1a1 1 0 1 0 0 2h.001a1 1 0 1 0 0-2H14z"+ />+ </svg>+ );+}++export { BugIcon };diff --git a/skyvern-frontend/src/components/icons/DebugIcon.tsx b/skyvern-frontend/src/components/icons/DebugIcon.tsxnew file mode 100644index 000000000..36fe67417--- /dev/null+++ b/skyvern-frontend/src/components/icons/DebugIcon.tsx@@ -0,0 +1,29 @@+import { PaperPlaneIcon } from "@radix-ui/react-icons";++import { cn } from "@/util/utils";+import { BugIcon } from "./BugIcon";+import { Tip } from "../Tip";++type Props = {+ className?: string;+ // --+ onClick?: () => void;+};++function DebugIcon({ className, onClick }: Props) {+ return (+ <Tip content="Debug (pre-convo-UI)">+ <div+ className={cn("relative flex items-center justify-center", className)}+ onClick={onClick}+ >+ <PaperPlaneIcon className="size-6 cursor-pointer" />+ <div className="absolute right-[-0.75rem] top-[-0.75rem] origin-center rotate-45 text-[#ff7e7e]">+ <BugIcon className="scale-75" />+ </div>+ </div>+ </Tip>+ );+}++export { DebugIcon };diff --git a/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx b/skyvern-frontend/src/routes/tasks/create/PromptBox.tsxindex 1a91c249b..5b2aafbaf 100644--- a/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx+++ b/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx@@ -1,10 +1,5 @@import { getClient } from "@/api/AxiosClient";-import {- Createv2TaskRequest,- TaskV2,- ProxyLocation,- TaskGenerationApiResponse,-} from "@/api/types";+import { Createv2TaskRequest, ProxyLocation } from "@/api/types";import img from "@/assets/promptBoxBg.png";import { AutoResizingTextarea } from "@/components/AutoResizingTextarea/AutoResizingTextarea";import { CartIcon } from "@/components/icons/CartIcon";@@ -26,6 +21,7 @@ import {import { Switch } from "@/components/ui/switch";import { toast } from "@/components/ui/use-toast";import { useCredentialGetter } from "@/hooks/useCredentialGetter";+import { WorkflowApiResponse } from "@/routes/workflows/types/workflowTypes";import { CodeEditor } from "@/routes/workflows/components/CodeEditor";import {FileTextIcon,@@ -38,7 +34,6 @@ import { useMutation, useQueryClient } from "@tanstack/react-query";import { AxiosError } from "axios";import { useState } from "react";import { useNavigate } from "react-router-dom";-import { stringify as convertToYAML } from "yaml";import {generatePhoneNumber,generateUniqueEmail,@@ -48,38 +43,7 @@ import {MAX_SCREENSHOT_SCROLLS_DEFAULT,MAX_STEPS_DEFAULT,} from "@/routes/workflows/editor/nodes/Taskv2Node/types";--function createTemplateTaskFromTaskGenerationParameters(- values: TaskGenerationApiResponse,-) {- return {- title: values.suggested_title ?? "Untitled Task",- description: "",- is_saved_task: true,- webhook_callback_url: null,- proxy_location: "RESIDENTIAL",- workflow_definition: {- parameters: [- {- parameter_type: "workflow",- workflow_parameter_type: "json",- key: "navigation_payload",- default_value: JSON.stringify(values.navigation_payload),- },- ],- blocks: [- {- block_type: "task",- label: values.suggested_title ?? "Untitled Task",- url: values.url,- navigation_goal: values.navigation_goal,- data_extraction_goal: values.data_extraction_goal,- data_schema: values.extracted_information_schema,- },- ],- },- };-}+import { useAutoplayStore } from "@/store/useAutoplayStore";const exampleCases = [{@@ -166,21 +130,22 @@ function PromptBox() {const [showAdvancedSettings, setShowAdvancedSettings] = useState(false);const [dataSchema, setDataSchema] = useState<string | null>(null);const [extraHttpHeaders, setExtraHttpHeaders] = useState<string | null>(null);+ const { setAutoplay } = useAutoplayStore();- const startObserverCruiseMutation = useMutation({+ const generateWorkflowMutation = useMutation({mutationFn: async (prompt: string) => {- const client = await getClient(credentialGetter, "v2");- return client.post<Createv2TaskRequest, { data: TaskV2 }>(- "/tasks",+ const client = await getClient(credentialGetter, "sans-api-v1");++ const result = await client.post<+ Createv2TaskRequest,+ { data: WorkflowApiResponse }+ >(+ "/workflows/create-from-prompt",{user_prompt: prompt,webhook_callback_url: webhookCallbackUrl,proxy_location: proxyLocation,- browser_session_id: browserSessionId,- browser_address: cdpAddress,totp_identifier: totpIdentifier,- generate_script: generateScript,- publish_workflow: publishWorkflow || generateScript,max_screenshot_scrolls: maxScreenshotScrolls,extracted_information_schema: dataSchema? (() => {@@ -207,76 +172,32 @@ function PromptBox() {},},);++ return result;},- onSuccess: (response) => {+ onSuccess: ({ data: workflow }) => {toast({variant: "success",- title: "Workflow Run Created",- description: `Workflow run created successfully.`,- });- queryClient.invalidateQueries({- queryKey: ["workflowRuns"],+ title: "Workflow Created",+ description: `Workflow created successfully.`,});+queryClient.invalidateQueries({queryKey: ["workflows"],});- queryClient.invalidateQueries({- queryKey: ["runs"],- });- navigate(- `/workflows/${response.data.workflow_permanent_id}/${response.data.workflow_run_id}`,- );- },- onError: (error: AxiosError) => {- toast({- variant: "destructive",- title: "Error creating workflow run from prompt",- description: error.message,- });- },- });- const getTaskFromPromptMutation = useMutation({- mutationFn: async (prompt: string) => {- const client = await getClient(credentialGetter);- return client- .post<- { prompt: string },- { data: TaskGenerationApiResponse }- >("/generate/task", { prompt })- .then((response) => response.data);- },- onError: (error: AxiosError) => {- const detail = (error.response?.data as { detail?: string })?.detail;- toast({- variant: "destructive",- title: "Error creating task from prompt",- description: detail ? detail : error.message,- });- },- });+ const firstBlock = workflow.workflow_definition.blocks[0];- const saveTaskMutation = useMutation({- mutationFn: async (params: TaskGenerationApiResponse) => {- const client = await getClient(credentialGetter);- const templateTask =- createTemplateTaskFromTaskGenerationParameters(params);- const yaml = convertToYAML(templateTask);- return client.post("/workflows", yaml, {- headers: {- "Content-Type": "text/plain",- },- });- },- onSuccess: () => {- queryClient.invalidateQueries({- queryKey: ["savedTasks"],- });+ if (firstBlock) {+ setAutoplay(workflow.workflow_permanent_id, firstBlock.label);+ }++ navigate(`/workflows/${workflow.workflow_permanent_id}/debug`);},onError: (error: AxiosError) => {toast({variant: "destructive",- title: "Error saving task",+ title: "Error creating workflow from prompt",description: error.message,});},@@ -343,30 +264,21 @@ function PromptBox() {/></div><div className="flex items-center">- {startObserverCruiseMutation.isPending ||- getTaskFromPromptMutation.isPending ||- saveTaskMutation.isPending ? (+ {generateWorkflowMutation.isPending ? (<ReloadIcon className="size-6 animate-spin" />) : (- <PaperPlaneIcon- className="size-6 cursor-pointer"- onClick={async () => {- if (selectValue === "v2") {- startObserverCruiseMutation.mutate(prompt);- return;- }- const taskGenerationResponse =- await getTaskFromPromptMutation.mutateAsync(prompt);- await saveTaskMutation.mutateAsync(- taskGenerationResponse,- );- navigate("/tasks/create/from-prompt", {- state: {- data: taskGenerationResponse,- },- });- }}- />+ <div className="flex items-center">+ {generateWorkflowMutation.isPending ? (+ <ReloadIcon className="size-6 animate-spin" />+ ) : (+ <PaperPlaneIcon+ className="size-6 cursor-pointer"+ onClick={async () => {+ generateWorkflowMutation.mutate(prompt);+ }}+ />+ )}+ </div>)}</div></div>diff --git a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsxindex 3992cefe8..ae8312499 100644--- a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx@@ -98,7 +98,7 @@ function Workspace({const { blockLabel, workflowPermanentId } = useParams();const [searchParams, setSearchParams] = useSearchParams();const cacheKeyValueParam = searchParams.get("cache-key-value");- const [timelineMode, setTimelineMode] = useState("narrow");+ const [timelineMode, setTimelineMode] = useState("wide");const [cacheKeyValueFilter, setCacheKeyValueFilter] = useState<string | null>(null,);diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/Taskv2Node/Taskv2Node.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/Taskv2Node/Taskv2Node.tsxindex 9bc3759f3..963d66540 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/Taskv2Node/Taskv2Node.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/Taskv2Node/Taskv2Node.tsx@@ -17,6 +17,7 @@ import { MAX_STEPS_DEFAULT, type Taskv2Node } from "./types";import { ModelSelector } from "@/components/ModelSelector";import { cn } from "@/util/utils";import { NodeHeader } from "../components/NodeHeader";+import { NodeFooter } from "../components/NodeFooter";import { useParams } from "react-router-dom";import { statusIsRunningOrQueued } from "@/routes/tasks/types";import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";@@ -195,6 +196,7 @@ function Taskv2Node({ id, data, type }: NodeProps<Taskv2Node>) {</AccordionContent></AccordionItem></Accordion>+ <NodeFooter blockLabel={label} /></div></div>);diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsxindex 835aba2d5..ace123897 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx@@ -11,6 +11,7 @@ import { toast } from "@/components/ui/use-toast";import { useLogging } from "@/hooks/useLogging";import { useCredentialGetter } from "@/hooks/useCredentialGetter";import { useOnChange } from "@/hooks/useOnChange";+import { useAutoplayStore } from "@/store/useAutoplayStore";import { useNodeLabelChangeHandler } from "@/routes/workflows/hooks/useLabelChangeHandler";import { useDeleteNodeCallback } from "@/routes/workflows/hooks/useDeleteNodeCallback";@@ -186,6 +187,28 @@ function NodeHeader({const [workflowRunStatus, setWorkflowRunStatus] = useState(workflowRun?.status,);+ const { getAutoplay, setAutoplay } = useAutoplayStore();++ useEffect(() => {+ if (!debugSession) {+ return;+ }++ const details = getAutoplay();++ if (+ workflowPermanentId === details.wpid &&+ blockLabel === details.blockLabel+ ) {+ setAutoplay(null, null);+ setTimeout(() => {+ runBlock.mutateAsync();+ }, 100);+ }++ // on mount+ // eslint-disable-next-line react-hooks/exhaustive-deps+ }, [debugSession]);useEffect(() => {setWorkflowRunStatus(workflowRun?.status);@@ -496,8 +519,10 @@ function NodeHeader({) : (<PlayIconclassName={cn("size-6", {- "fill-gray-500 text-gray-500":- workflowRunIsRunningOrQueued || !workflowPermanentId,+ "pointer-events-none fill-gray-500 text-gray-500":+ workflowRunIsRunningOrQueued ||+ !workflowPermanentId ||+ debugSession === undefined,})}onClick={() => {handleOnPlay();diff --git a/skyvern-frontend/src/routes/workflows/hooks/useCacheKeyValuesQuery.ts b/skyvern-frontend/src/routes/workflows/hooks/useCacheKeyValuesQuery.tsindex c9b28139f..bd3f13132 100644--- a/skyvern-frontend/src/routes/workflows/hooks/useCacheKeyValuesQuery.ts+++ b/skyvern-frontend/src/routes/workflows/hooks/useCacheKeyValuesQuery.ts@@ -56,7 +56,7 @@ function useCacheKeyValuesQuery({return result;},- enabled: !!workflowPermanentId,+ enabled: !!workflowPermanentId && !!cacheKey && cacheKey.length > 0,placeholderData: keepPreviousData,staleTime: 5 * 60 * 1000,});diff --git a/skyvern-frontend/src/store/useAutoplayStore.ts b/skyvern-frontend/src/store/useAutoplayStore.tsnew file mode 100644index 000000000..d8102085d--- /dev/null+++ b/skyvern-frontend/src/store/useAutoplayStore.ts@@ -0,0 +1,24 @@+import { create } from "zustand";++type AutoplayStore = {+ wpid: string | null;+ blockLabel: string | null;+ setAutoplay: (wpid: string | null, blockLabel: string | null) => void;+ clearAutoplay: () => void;+ getAutoplay: () => { wpid: string | null; blockLabel: string | null };+};++export const useAutoplayStore = create<AutoplayStore>((set, get) => ({+ wpid: null,+ blockLabel: null,+ setAutoplay: (wpid: string | null, blockLabel: string | null) => {+ set({ wpid, blockLabel });+ },+ clearAutoplay: () => {+ set({ wpid: null, blockLabel: null });+ },+ getAutoplay: () => {+ const { wpid, blockLabel } = get();+ return { wpid, blockLabel };+ },+}));