Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Add observer in UI (#1409)
Shuchang Zheng69687f121ad7
diff --git a/skyvern-frontend/src/api/types.ts b/skyvern-frontend/src/api/types.tsindex d9c00a614..a897e2dd6 100644--- a/skyvern-frontend/src/api/types.ts+++ b/skyvern-frontend/src/api/types.ts@@ -233,3 +233,15 @@ export type ActionsApiResponse = {intention: string | null;response: string | null;};++export type ObserverCruise = {+ observer_cruise_id: string;+ status: Status;+ workflow_run_id: string | null;+ workflow_id: string | null;+ workflow_permanent_id: string | null;+ prompt: string | null;+ url: string | null;+ created_at: string;+ modified_at: string;+};diff --git a/skyvern-frontend/src/components/ui/select.tsx b/skyvern-frontend/src/components/ui/select.tsxindex c6bd55820..0edc69496 100644--- a/skyvern-frontend/src/components/ui/select.tsx+++ b/skyvern-frontend/src/components/ui/select.tsx@@ -114,6 +114,30 @@ const SelectLabel = React.forwardRef<));SelectLabel.displayName = SelectPrimitive.Label.displayName;+const CustomSelectItem = React.forwardRef<+ React.ElementRef<typeof SelectPrimitive.Item>,+ React.ComponentPropsWithoutRef<typeof SelectPrimitive.Item>+>(({ className, children, ...props }, ref) => (+ <SelectPrimitive.Item+ ref={ref}+ className={cn(+ "relative flex w-full cursor-default select-none items-center rounded-sm py-1.5 pl-2 pr-8 text-sm outline-none focus:bg-accent focus:text-accent-foreground data-[disabled]:pointer-events-none data-[disabled]:opacity-50",+ className,+ )}+ {...props}+ >+ <span className="absolute right-2 flex h-3.5 w-3.5 items-center justify-center">+ <SelectPrimitive.ItemIndicator>+ <CheckIcon className="h-4 w-4" />+ </SelectPrimitive.ItemIndicator>+ </span>+ {children}+ </SelectPrimitive.Item>+));+CustomSelectItem.displayName = SelectPrimitive.Item.displayName;++const SelectItemText = SelectPrimitive.ItemText;+const SelectItem = React.forwardRef<React.ElementRef<typeof SelectPrimitive.Item>,React.ComponentPropsWithoutRef<typeof SelectPrimitive.Item>@@ -159,4 +183,6 @@ export {SelectSeparator,SelectScrollUpButton,SelectScrollDownButton,+ CustomSelectItem,+ SelectItemText,};diff --git a/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx b/skyvern-frontend/src/routes/tasks/create/PromptBox.tsxindex f3f8913e2..1a538cb1f 100644--- a/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx+++ b/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx@@ -1,5 +1,5 @@import { getClient } from "@/api/AxiosClient";-import { TaskGenerationApiResponse } from "@/api/types";+import { ObserverCruise, TaskGenerationApiResponse } from "@/api/types";import img from "@/assets/promptBoxBg.png";import { CartIcon } from "@/components/icons/CartIcon";import { GraphIcon } from "@/components/icons/GraphIcon";@@ -7,9 +7,19 @@ import { InboxIcon } from "@/components/icons/InboxIcon";import { MessageIcon } from "@/components/icons/MessageIcon";import { TranslateIcon } from "@/components/icons/TranslateIcon";import { TrophyIcon } from "@/components/icons/TrophyIcon";+import { Button } from "@/components/ui/button";+import {+ CustomSelectItem,+ Select,+ SelectContent,+ SelectItemText,+ SelectTrigger,+ SelectValue,+} from "@/components/ui/select";import { Textarea } from "@/components/ui/textarea";import { toast } from "@/components/ui/use-toast";import { useCredentialGetter } from "@/hooks/useCredentialGetter";+import { observerFeatureEnabled } from "@/util/env";import {FileTextIcon,GearIcon,@@ -18,10 +28,11 @@ import {PlusIcon,ReloadIcon,} from "@radix-ui/react-icons";+import { ToastAction } from "@radix-ui/react-toast";import { useMutation, useQueryClient } from "@tanstack/react-query";import { AxiosError } from "axios";import { useState } from "react";-import { useNavigate } from "react-router-dom";+import { Link, useNavigate } from "react-router-dom";import { stringify as convertToYAML } from "yaml";function createTemplateTaskFromTaskGenerationParameters(@@ -112,9 +123,53 @@ const exampleCases = [function PromptBox() {const navigate = useNavigate();const [prompt, setPrompt] = useState<string>("");+ const [selectValue, setSelectValue] = useState(+ observerFeatureEnabled ? "v2" : "v1",+ );const credentialGetter = useCredentialGetter();const queryClient = useQueryClient();+ const startObserverCruiseMutation = useMutation({+ mutationFn: async (prompt: string) => {+ const client = await getClient(credentialGetter);+ return client.post<{ user_prompt: string }, { data: ObserverCruise }>(+ "/cruise",+ { user_prompt: prompt },+ );+ },+ onSuccess: (response) => {+ toast({+ variant: "success",+ title: "Workflow Run Created",+ description: `Workflow run created successfully.`,+ action: (+ <ToastAction altText="View">+ <Button asChild>+ <Link+ to={`/workflows/${response.data.workflow_permanent_id}/${response.data.workflow_run_id}`}+ >+ View+ </Link>+ </Button>+ </ToastAction>+ ),+ });+ queryClient.invalidateQueries({+ queryKey: ["workflowRuns"],+ });+ queryClient.invalidateQueries({+ queryKey: ["workflows"],+ });+ },+ onError: (error: AxiosError) => {+ toast({+ variant: "destructive",+ title: "Error creating workflow run from prompt",+ description: error.message,+ });+ },+ });+const getTaskFromPromptMutation = useMutation({mutationFn: async (prompt: string) => {const client = await getClient(credentialGetter);@@ -180,14 +235,48 @@ function PromptBox() {placeholder="Enter your prompt..."rows={1}/>- <div className="h-full">- {getTaskFromPromptMutation.isPending ||+ {observerFeatureEnabled && (+ <Select value={selectValue} onValueChange={setSelectValue}>+ <SelectTrigger className="w-32 focus:ring-0">+ <SelectValue />+ </SelectTrigger>+ <SelectContent className="border-slate-500 bg-slate-elevation3">+ <CustomSelectItem value="v2" className="hover:bg-slate-800">+ <div className="space-y-2">+ <div>+ <SelectItemText>Observer</SelectItemText>+ </div>+ <div className="text-xs text-slate-400">+ best for complex tasks+ </div>+ </div>+ </CustomSelectItem>+ <CustomSelectItem value="v1">+ <div className="space-y-2">+ <div>+ <SelectItemText>Task</SelectItemText>+ </div>+ <div className="text-xs text-slate-400">+ best for simple tasks+ </div>+ </div>+ </CustomSelectItem>+ </SelectContent>+ </Select>+ )}+ <div className="flex items-center">+ {startObserverCruiseMutation.isPending ||+ getTaskFromPromptMutation.isPending ||saveTaskMutation.isPending ? (<ReloadIcon className="h-6 w-6 animate-spin" />) : (<PaperPlaneIconclassName="h-6 w-6 cursor-pointer"onClick={async () => {+ if (observerFeatureEnabled && selectValue === "v2") {+ startObserverCruiseMutation.mutate(prompt);+ return;+ }const taskGenerationResponse =await getTaskFromPromptMutation.mutateAsync(prompt);await saveTaskMutation.mutateAsync(taskGenerationResponse);diff --git a/skyvern-frontend/src/util/env.ts b/skyvern-frontend/src/util/env.tsindex d54adc000..19721eb63 100644--- a/skyvern-frontend/src/util/env.ts+++ b/skyvern-frontend/src/util/env.ts@@ -19,4 +19,13 @@ if (!artifactApiBaseUrl) {console.warn("artifactApiBaseUrl environment variable was not set");}-export { apiBaseUrl, environment, envCredential, artifactApiBaseUrl };+const observerEnabled = import.meta.env.VITE_OBSERVER_ENABLED as string;+const observerFeatureEnabled = observerEnabled === "true";++export {+ apiBaseUrl,+ environment,+ envCredential,+ artifactApiBaseUrl,+ observerFeatureEnabled,+};