Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Task creation UI updates (#886)
Kerem Yilmaz0b8f6ac1d844
diff --git a/skyvern-frontend/src/assets/promptBoxBg.png b/skyvern-frontend/src/assets/promptBoxBg.pngnew file mode 100644index 000000000..82a4e8b0cBinary files /dev/null and b/skyvern-frontend/src/assets/promptBoxBg.png differdiff --git a/skyvern-frontend/src/components/SwitchBar.tsx b/skyvern-frontend/src/components/SwitchBar.tsxnew file mode 100644index 000000000..2a0e3fdab--- /dev/null+++ b/skyvern-frontend/src/components/SwitchBar.tsx@@ -0,0 +1,42 @@+import { cn } from "@/util/utils";++type Option = {+ label: string;+ value: string;+};++type Props = {+ options: Option[];+ value: string;+ onChange: (value: string) => void;+};++function SwitchBar({ options, value, onChange }: Props) {+ return (+ <div className="flex w-fit gap-1 rounded-sm border border-slate-700 p-2">+ {options.map((option) => {+ const selected = option.value === value;+ return (+ <div+ key={option.value}+ className={cn(+ "cursor-pointer rounded-sm px-3 py-2 text-xs hover:bg-slate-700",+ {+ "bg-slate-700": selected,+ },+ )}+ onClick={() => {+ if (!selected) {+ onChange(option.value);+ }+ }}+ >+ {option.label}+ </div>+ );+ })}+ </div>+ );+}++export { SwitchBar };diff --git a/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx b/skyvern-frontend/src/routes/tasks/create/PromptBox.tsxnew file mode 100644index 000000000..56705ed33--- /dev/null+++ b/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx@@ -0,0 +1,200 @@+import { getClient } from "@/api/AxiosClient";+import { TaskGenerationApiResponse } from "@/api/types";+import img from "@/assets/promptBoxBg.png";+import { Textarea } from "@/components/ui/textarea";+import { toast } from "@/components/ui/use-toast";+import { useCredentialGetter } from "@/hooks/useCredentialGetter";+import { PaperPlaneIcon, ReloadIcon } from "@radix-ui/react-icons";+import { useMutation, useQueryClient } from "@tanstack/react-query";+import { AxiosError } from "axios";+import { useState } from "react";+import { useNavigate } from "react-router-dom";+import { stringify as convertToYAML } from "yaml";++function createTaskFromTaskGenerationParameters(+ values: TaskGenerationApiResponse,+) {+ return {+ url: values.url,+ navigation_goal: values.navigation_goal,+ data_extraction_goal: values.data_extraction_goal,+ proxy_location: "RESIDENTIAL",+ navigation_payload: values.navigation_payload,+ extracted_information_schema: values.extracted_information_schema,+ };+}++function createTemplateTaskFromTaskGenerationParameters(+ values: TaskGenerationApiResponse,+) {+ return {+ title: values.suggested_title ?? "Untitled Task",+ description: "",+ is_saved_task: true,+ webhook_callback_url: null,+ proxy_location: "RESIDENTIAL",+ workflow_definition: {+ parameters: [+ {+ parameter_type: "workflow",+ workflow_parameter_type: "json",+ key: "navigation_payload",+ default_value: JSON.stringify(values.navigation_payload),+ },+ ],+ blocks: [+ {+ block_type: "task",+ label: values.suggested_title ?? "Untitled Task",+ url: values.url,+ navigation_goal: values.navigation_goal,+ data_extraction_goal: values.data_extraction_goal,+ data_schema: values.extracted_information_schema,+ },+ ],+ },+ };+}++const examplePrompts = [+ "What is the top post on hackernews?",+ "Navigate to Google Finance and search for AAPL",+];++function PromptBox() {+ const navigate = useNavigate();+ const [prompt, setPrompt] = useState<string>("");+ const credentialGetter = useCredentialGetter();+ const queryClient = useQueryClient();++ const getTaskFromPromptMutation = useMutation({+ mutationFn: async (prompt: string) => {+ const client = await getClient(credentialGetter);+ return client+ .post<+ { prompt: string },+ { data: TaskGenerationApiResponse }+ >("/generate/task", { prompt })+ .then((response) => response.data);+ },+ onError: (error: AxiosError) => {+ toast({+ variant: "destructive",+ title: "Error creating task from prompt",+ description: error.message,+ });+ },+ });++ const saveTaskMutation = useMutation({+ mutationFn: async (params: TaskGenerationApiResponse) => {+ const client = await getClient(credentialGetter);+ const templateTask =+ createTemplateTaskFromTaskGenerationParameters(params);+ const yaml = convertToYAML(templateTask);+ return client.post("/workflows", yaml, {+ headers: {+ "Content-Type": "text/plain",+ },+ });+ },+ onSuccess: () => {+ queryClient.invalidateQueries({+ queryKey: ["savedTasks"],+ });+ },+ onError: (error: AxiosError) => {+ toast({+ variant: "destructive",+ title: "Error saving task",+ description: error.message,+ });+ },+ });++ const runTaskMutation = useMutation({+ mutationFn: async (params: TaskGenerationApiResponse) => {+ const client = await getClient(credentialGetter);+ const data = createTaskFromTaskGenerationParameters(params);+ return client.post<+ ReturnType<typeof createTaskFromTaskGenerationParameters>,+ { data: { task_id: string } }+ >("/tasks", data);+ },+ onSuccess: (response) => {+ navigate(`/tasks/${response.data.task_id}/actions`);+ },+ onError: (error: AxiosError) => {+ toast({+ variant: "destructive",+ title: "Error running task",+ description: error.message,+ });+ },+ });++ return (+ <div>+ <div+ className="rounded-sm py-[4.25rem]"+ style={{+ background: `url(${img}) 50% / cover no-repeat`,+ }}+ >+ <div className="flex flex-col items-center gap-7">+ <span className="text-2xl">+ What task would you like to accomplish?+ </span>+ <div+ className="flex w-[35rem] max-w-xl items-center rounded-xl border border-slate-700 py-2 pr-4"+ style={{+ background: "rgba(248, 250, 252, 0.05)",+ }}+ >+ <Textarea+ className="min-h-0 resize-none rounded-xl border-transparent px-4 text-slate-400 hover:border-transparent focus-visible:ring-0"+ value={prompt}+ onChange={(e) => setPrompt(e.target.value)}+ placeholder="Enter your prompt..."+ rows={1}+ />+ <div className="h-full">+ {getTaskFromPromptMutation.isPending ||+ saveTaskMutation.isPending ||+ runTaskMutation.isPending ? (+ <ReloadIcon className="h-6 w-6 animate-spin" />+ ) : (+ <PaperPlaneIcon+ className="h-6 w-6 cursor-pointer"+ onClick={async () => {+ const taskGenerationResponse =+ await getTaskFromPromptMutation.mutateAsync(prompt);+ await saveTaskMutation.mutateAsync(taskGenerationResponse);+ await runTaskMutation.mutateAsync(taskGenerationResponse);+ }}+ />+ )}+ </div>+ </div>+ </div>+ </div>+ <div className="flex gap-4 rounded-sm bg-slate-elevation1 p-4">+ {examplePrompts.map((examplePrompt) => {+ return (+ <div+ key={examplePrompt}+ className="cursor-pointer rounded-sm bg-slate-elevation3 px-4 py-3 hover:bg-slate-elevation5"+ onClick={() => {+ setPrompt(examplePrompt);+ }}+ >+ {examplePrompt}+ </div>+ );+ })}+ </div>+ </div>+ );+}++export { PromptBox };diff --git a/skyvern-frontend/src/routes/tasks/create/SavedTaskCard.tsx b/skyvern-frontend/src/routes/tasks/create/SavedTaskCard.tsxindex 59a5de650..0679a7bef 100644--- a/skyvern-frontend/src/routes/tasks/create/SavedTaskCard.tsx+++ b/skyvern-frontend/src/routes/tasks/create/SavedTaskCard.tsx@@ -14,7 +14,6 @@ import {DialogFooter,DialogHeader,DialogTitle,- DialogTrigger,} from "@/components/ui/dialog";import {DropdownMenu,@@ -26,6 +25,7 @@ import {} from "@/components/ui/dropdown-menu";import { toast } from "@/components/ui/use-toast";import { useCredentialGetter } from "@/hooks/useCredentialGetter";+import { cn } from "@/util/utils";import { DotsHorizontalIcon, ReloadIcon } from "@radix-ui/react-icons";import { useMutation, useQueryClient } from "@tanstack/react-query";import { useState } from "react";@@ -43,6 +43,7 @@ function SavedTaskCard({ workflowId, title, url, description }: Props) {const credentialGetter = useCredentialGetter();const queryClient = useQueryClient();const navigate = useNavigate();+ const [hovering, setHovering] = useState(false);const deleteTaskMutation = useMutation({mutationFn: async (id: string) => {@@ -73,29 +74,42 @@ function SavedTaskCard({ workflowId, title, url, description }: Props) {});return (- <Card>- <CardHeader>- <CardTitle className="flex items-center justify-between">+ <Card+ className="border-0"+ onMouseEnter={() => setHovering(true)}+ onMouseLeave={() => setHovering(false)}+ onMouseOver={() => setHovering(true)}+ onMouseOut={() => setHovering(false)}+ >+ <CardHeader+ className={cn("rounded-t-md bg-slate-elevation1", {+ "bg-slate-900": hovering,+ })}+ >+ <CardTitle className="flex items-center justify-between font-normal"><span className="overflow-hidden text-ellipsis whitespace-nowrap">{title}</span>- <Dialog open={open} onOpenChange={setOpen}>- <DropdownMenu>+ <Dialog+ open={open}+ onOpenChange={() => {+ setOpen(false);+ }}+ >+ <DropdownMenu modal={false}><DropdownMenuTrigger asChild><DotsHorizontalIcon className="cursor-pointer" /></DropdownMenuTrigger><DropdownMenuContent className="w-56"><DropdownMenuLabel>Template Actions</DropdownMenuLabel><DropdownMenuSeparator />- <DialogTrigger asChild>- <DropdownMenuItem- onSelect={() => {- setOpen(true);- }}- >- Delete Template- </DropdownMenuItem>- </DialogTrigger>+ <DropdownMenuItem+ onSelect={() => {+ setOpen(true);+ }}+ >+ Delete Template+ </DropdownMenuItem></DropdownMenuContent></DropdownMenu><DialogContent>@@ -106,7 +120,12 @@ function SavedTaskCard({ workflowId, title, url, description }: Props) {</DialogDescription></DialogHeader><DialogFooter>- <Button variant="secondary" onClick={() => setOpen(false)}>+ <Button+ variant="secondary"+ onClick={() => {+ setOpen(false);+ }}+ >Cancel</Button><Button@@ -125,12 +144,17 @@ function SavedTaskCard({ workflowId, title, url, description }: Props) {</DialogContent></Dialog></CardTitle>- <CardDescription className="overflow-hidden text-ellipsis whitespace-nowrap">+ <CardDescription className="overflow-hidden text-ellipsis whitespace-nowrap text-slate-400">{url}</CardDescription></CardHeader><CardContent- className="h-48 cursor-pointer overflow-scroll hover:bg-muted/40"+ className={cn(+ "h-36 cursor-pointer overflow-scroll rounded-b-md bg-slate-elevation3 p-4 text-sm text-slate-300",+ {+ "bg-slate-800": hovering,+ },+ )}onClick={() => {navigate(workflowId);}}diff --git a/skyvern-frontend/src/routes/tasks/create/SavedTasks.tsx b/skyvern-frontend/src/routes/tasks/create/SavedTasks.tsxindex c7531a921..2b9011ec0 100644--- a/skyvern-frontend/src/routes/tasks/create/SavedTasks.tsx+++ b/skyvern-frontend/src/routes/tasks/create/SavedTasks.tsx@@ -15,6 +15,8 @@ import { useMutation, useQuery } from "@tanstack/react-query";import { useNavigate } from "react-router-dom";import { stringify as convertToYAML } from "yaml";import { SavedTaskCard } from "./SavedTaskCard";+import { useState } from "react";+import { cn } from "@/util/utils";function createEmptyTaskTemplate() {return {@@ -49,6 +51,7 @@ function createEmptyTaskTemplate() {function SavedTasks() {const credentialGetter = useCredentialGetter();const navigate = useNavigate();+ const [hovering, setHovering] = useState(false);const { data } = useQuery<Array<WorkflowApiResponse>>({queryKey: ["savedTasks"],@@ -99,20 +102,36 @@ function SavedTasks() {return (<div className="grid grid-cols-4 gap-4"><Card- onClick={() => {- if (mutation.isPending) {- return;- }- mutation.mutate();- }}+ className="border-0"+ onMouseEnter={() => setHovering(true)}+ onMouseLeave={() => setHovering(false)}+ onMouseOver={() => setHovering(true)}+ onMouseOut={() => setHovering(false)}>- <CardHeader>- <CardTitle>New Template</CardTitle>- <CardDescription className="overflow-hidden text-ellipsis whitespace-nowrap">+ <CardHeader+ className={cn("rounded-t-md bg-slate-elevation1", {+ "bg-slate-900": hovering,+ })}+ >+ <CardTitle className="font-normal">New Template</CardTitle>+ <CardDescription className="overflow-hidden text-ellipsis whitespace-nowrap text-slate-400">Create your own template</CardDescription></CardHeader>- <CardContent className="flex h-48 cursor-pointer items-center justify-center hover:bg-muted/40">+ <CardContent+ className={cn(+ "flex h-36 cursor-pointer items-center justify-center rounded-b-md bg-slate-elevation3 p-4 text-sm text-slate-300",+ {+ "bg-slate-800": hovering,+ },+ )}+ onClick={() => {+ if (mutation.isPending) {+ return;+ }+ mutation.mutate();+ }}+ >{!mutation.isPending && <PlusIcon className="h-12 w-12" />}{mutation.isPending && (<ReloadIcon className="h-12 w-12 animate-spin" />diff --git a/skyvern-frontend/src/routes/tasks/create/TaskTemplateCard.tsx b/skyvern-frontend/src/routes/tasks/create/TaskTemplateCard.tsxnew file mode 100644index 000000000..0a735df55--- /dev/null+++ b/skyvern-frontend/src/routes/tasks/create/TaskTemplateCard.tsx@@ -0,0 +1,56 @@+import {+ Card,+ CardContent,+ CardDescription,+ CardHeader,+ CardTitle,+} from "@/components/ui/card";+import { cn } from "@/util/utils";+import { useState } from "react";++type Props = {+ title: string;+ description: string;+ body: string;+ onClick: () => void;+};++function TaskTemplateCard({ title, description, body, onClick }: Props) {+ const [hovering, setHovering] = useState(false);++ return (+ <Card+ className="border-0"+ onMouseEnter={() => setHovering(true)}+ onMouseLeave={() => setHovering(false)}+ onMouseOver={() => setHovering(true)}+ onMouseOut={() => setHovering(false)}+ >+ <CardHeader+ className={cn("rounded-t-md bg-slate-elevation1", {+ "bg-slate-900": hovering,+ })}+ >+ <CardTitle className="font-normal">{title}</CardTitle>+ <CardDescription className="overflow-hidden text-ellipsis whitespace-nowrap text-slate-400">+ {description}+ </CardDescription>+ </CardHeader>+ <CardContent+ className={cn(+ "h-36 cursor-pointer rounded-b-md bg-slate-elevation3 p-4 text-sm text-slate-300",+ {+ "bg-slate-800": hovering,+ },+ )}+ onClick={() => {+ onClick();+ }}+ >+ {body}+ </CardContent>+ </Card>+ );+}++export { TaskTemplateCard };diff --git a/skyvern-frontend/src/routes/tasks/create/TaskTemplates.tsx b/skyvern-frontend/src/routes/tasks/create/TaskTemplates.tsxindex 5f6db0667..eb43a5c1f 100644--- a/skyvern-frontend/src/routes/tasks/create/TaskTemplates.tsx+++ b/skyvern-frontend/src/routes/tasks/create/TaskTemplates.tsx@@ -1,35 +1,11 @@-import { SampleCase } from "../types";-import {- Card,- CardContent,- CardDescription,- CardHeader,- CardTitle,-} from "@/components/ui/card";-import { Separator } from "@/components/ui/separator";import { useNavigate } from "react-router-dom";-import { SavedTasks } from "./SavedTasks";import { getSample } from "../data/sampleTaskData";-import { Textarea } from "@/components/ui/textarea";+import { SampleCase } from "../types";+import { PromptBox } from "./PromptBox";+import { SavedTasks } from "./SavedTasks";+import { SwitchBar } from "@/components/SwitchBar";import { useState } from "react";-import {- InfoCircledIcon,- PaperPlaneIcon,- ReloadIcon,-} from "@radix-ui/react-icons";-import { useMutation, useQueryClient } from "@tanstack/react-query";-import { useCredentialGetter } from "@/hooks/useCredentialGetter";-import { getClient } from "@/api/AxiosClient";-import { AxiosError } from "axios";-import { toast } from "@/components/ui/use-toast";-import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";-import { TaskGenerationApiResponse } from "@/api/types";-import { stringify as convertToYAML } from "yaml";--const examplePrompts = [- "What is the top post on hackernews?",- "Navigate to Google Finance and search for AAPL",-];+import { TaskTemplateCard } from "./TaskTemplateCard";const templateSamples: {[key in SampleCase]: {@@ -63,228 +39,52 @@ const templateSamples: {},};-function createTemplateTaskFromTaskGenerationParameters(- values: TaskGenerationApiResponse,-) {- return {- title: values.suggested_title ?? "Untitled Task",- description: "",- is_saved_task: true,- webhook_callback_url: null,- proxy_location: "RESIDENTIAL",- workflow_definition: {- parameters: [- {- parameter_type: "workflow",- workflow_parameter_type: "json",- key: "navigation_payload",- default_value: JSON.stringify(values.navigation_payload),- },- ],- blocks: [- {- block_type: "task",- label: values.suggested_title ?? "Untitled Task",- url: values.url,- navigation_goal: values.navigation_goal,- data_extraction_goal: values.data_extraction_goal,- data_schema: values.extracted_information_schema,- },- ],- },- };-}--function createTaskFromTaskGenerationParameters(- values: TaskGenerationApiResponse,-) {- return {- url: values.url,- navigation_goal: values.navigation_goal,- data_extraction_goal: values.data_extraction_goal,- proxy_location: "RESIDENTIAL",- navigation_payload: values.navigation_payload,- extracted_information_schema: values.extracted_information_schema,- };-}+const templateSwitchOptions = [+ {+ label: "Skyvern Templates",+ value: "skyvern",+ },+ {+ label: "My Templates",+ value: "user",+ },+];function TaskTemplates() {const navigate = useNavigate();- const [prompt, setPrompt] = useState<string>("");- const credentialGetter = useCredentialGetter();- const queryClient = useQueryClient();-- const getTaskFromPromptMutation = useMutation({- mutationFn: async (prompt: string) => {- const client = await getClient(credentialGetter);- return client- .post<- { prompt: string },- { data: TaskGenerationApiResponse }- >("/generate/task", { prompt })- .then((response) => response.data);- },- onError: (error: AxiosError) => {- toast({- variant: "destructive",- title: "Error creating task from prompt",- description: error.message,- });- },- });-- const saveTaskMutation = useMutation({- mutationFn: async (params: TaskGenerationApiResponse) => {- const client = await getClient(credentialGetter);- const templateTask =- createTemplateTaskFromTaskGenerationParameters(params);- const yaml = convertToYAML(templateTask);- return client.post("/workflows", yaml, {- headers: {- "Content-Type": "text/plain",- },- });- },- onSuccess: () => {- queryClient.invalidateQueries({- queryKey: ["savedTasks"],- });- },- onError: (error: AxiosError) => {- toast({- variant: "destructive",- title: "Error saving task",- description: error.message,- });- },- });-- const runTaskMutation = useMutation({- mutationFn: async (params: TaskGenerationApiResponse) => {- const client = await getClient(credentialGetter);- const data = createTaskFromTaskGenerationParameters(params);- return client.post<- ReturnType<typeof createTaskFromTaskGenerationParameters>,- { data: { task_id: string } }- >("/tasks", data);- },- onSuccess: (response) => {- navigate(`/tasks/${response.data.task_id}/actions`);- },- onError: (error: AxiosError) => {- toast({- variant: "destructive",- title: "Error running task",- description: error.message,- });- },- });+ const [templateSwitchValue, setTemplateSwitchValue] =+ useState<(typeof templateSwitchOptions)[number]["value"]>("skyvern");return (- <div>- <Alert variant="warning">- <InfoCircledIcon className="h-4 w-4" />- <AlertTitle>- Have a complicated workflow you would like to automate?- </AlertTitle>- <AlertDescription>- <a- href="https://meetings.hubspot.com/skyvern/demo"- target="_blank"- rel="noopener noreferrer"- className="ml-auto underline underline-offset-2"- >- Book a demo {"->"}- </a>- </AlertDescription>- </Alert>- <section className="py-4">- <header>- <h1 className="mb-2 text-3xl">Try a prompt</h1>- </header>- <p className="text-sm">- We will generate a task for you automatically.- </p>- <Separator className="mb-8 mt-2" />- <div className="mx-auto flex max-w-xl items-center rounded-xl border pr-4">- <Textarea- className="resize-none rounded-xl border-transparent p-2 font-mono text-sm hover:border-transparent focus-visible:ring-0"- value={prompt}- onChange={(e) => setPrompt(e.target.value)}- placeholder="Enter your prompt..."- />- <div className="h-full">- {getTaskFromPromptMutation.isPending ||- saveTaskMutation.isPending ||- runTaskMutation.isPending ? (- <ReloadIcon className="h-6 w-6 animate-spin" />- ) : (- <PaperPlaneIcon- className="h-6 w-6 cursor-pointer"- onClick={async () => {- const taskGenerationResponse =- await getTaskFromPromptMutation.mutateAsync(prompt);- await saveTaskMutation.mutateAsync(taskGenerationResponse);- await runTaskMutation.mutateAsync(taskGenerationResponse);- }}- />- )}- </div>- </div>- <div className="mt-4 flex flex-wrap justify-center gap-4">- {examplePrompts.map((examplePrompt) => {- return (- <div- key={examplePrompt}- className="cursor-pointer rounded-xl border p-2 text-sm text-muted-foreground"- onClick={() => {- setPrompt(examplePrompt);- }}- >- {examplePrompt}- </div>- );- })}- </div>+ <div className="space-y-8">+ <PromptBox />+ <section>+ <SwitchBar+ value={templateSwitchValue}+ onChange={setTemplateSwitchValue}+ options={templateSwitchOptions}+ /></section>- <section className="py-4">- <header>- <h1 className="text-3xl">Your Templates</h1>- </header>- <p className="mt-1 text-sm">Your saved task templates</p>- <Separator className="mb-8 mt-2" />- <SavedTasks />- </section>- <section className="py-4">- <header>- <h1 className="text-3xl">Skyvern Templates</h1>- </header>- <p className="mt-1 text-sm">- Sample tasks that showcase Skyvern's capabilities- </p>- <Separator className="mb-8 mt-2" />- <div className="grid grid-cols-4 gap-4">- {Object.entries(templateSamples).map(([sampleKey, sample]) => {- return (- <Card key={sampleKey}>- <CardHeader>- <CardTitle>{sample.title}</CardTitle>- <CardDescription className="overflow-hidden text-ellipsis whitespace-nowrap">- {getSample(sampleKey as SampleCase).url}- </CardDescription>- </CardHeader>- <CardContent- className="h-48 cursor-pointer hover:bg-muted/40"+ <section>+ {templateSwitchValue === "skyvern" ? (+ <div className="grid grid-cols-4 gap-4">+ {Object.entries(templateSamples).map(([sampleKey, sample]) => {+ return (+ <TaskTemplateCard+ key={sampleKey}+ title={sample.title}+ description={getSample(sampleKey as SampleCase).url}+ body={sample.description}onClick={() => {- navigate(sampleKey);+ navigate(`/create/${sampleKey}`);}}- >- {sample.description}- </CardContent>- </Card>- );- })}- </div>+ />+ );+ })}+ </div>+ ) : (+ <SavedTasks />+ )}</section></div>);