Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Magic Wand Button to improve prompts in all blocks (#4060)
Celal Zamanoglu0cbab39a2724
diff --git a/skyvern-frontend/src/routes/workflows/editor/constants.ts b/skyvern-frontend/src/routes/workflows/editor/constants.tsindex 7da2dfd33..4793dd213 100644--- a/skyvern-frontend/src/routes/workflows/editor/constants.ts+++ b/skyvern-frontend/src/routes/workflows/editor/constants.ts@@ -20,3 +20,95 @@ export const BITWARDEN_CLIENT_SECRET_AWS_SECRET_KEY ="SKYVERN_BITWARDEN_CLIENT_SECRET";export const BITWARDEN_MASTER_PASSWORD_AWS_SECRET_KEY ="SKYVERN_BITWARDEN_MASTER_PASSWORD";++type AiImproveConfig = {+ useCase: string;+ context: Record<string, unknown>;+};++const createAiImproveConfig = (+ block: string,+ field: string,+ extraContext: Record<string, unknown> = {},+): AiImproveConfig => ({+ useCase: `workflow_editor.${block}.${field}`,+ context: {+ block_type: block,+ field,+ ...extraContext,+ },+});++export const AI_IMPROVE_CONFIGS = {+ task: {+ navigationGoal: createAiImproveConfig("task", "navigation_goal"),+ dataExtractionGoal: createAiImproveConfig("task", "data_extraction_goal"),+ completeCriterion: createAiImproveConfig("task", "complete_criterion"),+ },+ action: {+ navigationGoal: createAiImproveConfig("action", "navigation_goal"),+ errorCodeMapping: createAiImproveConfig("action", "error_code_mapping"),+ },+ navigation: {+ navigationGoal: createAiImproveConfig("navigation", "navigation_goal"),+ completeCriterion: createAiImproveConfig(+ "navigation",+ "complete_criterion",+ ),+ },+ extraction: {+ dataExtractionGoal: createAiImproveConfig(+ "extraction",+ "data_extraction_goal",+ ),+ dataSchema: createAiImproveConfig("extraction", "data_schema"),+ },+ validation: {+ completeCriterion: createAiImproveConfig(+ "validation",+ "complete_criterion",+ ),+ terminateCriterion: createAiImproveConfig(+ "validation",+ "terminate_criterion",+ ),+ },+ login: {+ navigationGoal: createAiImproveConfig("login", "navigation_goal"),+ completeCriterion: createAiImproveConfig("login", "complete_criterion"),+ terminateCriterion: createAiImproveConfig("login", "terminate_criterion"),+ },+ fileDownload: {+ navigationGoal: createAiImproveConfig("file_download", "navigation_goal"),+ completeCriterion: createAiImproveConfig(+ "file_download",+ "complete_criterion",+ ),+ },+ taskV2: {+ prompt: createAiImproveConfig("task_v2", "prompt"),+ },+ textPrompt: {+ prompt: createAiImproveConfig("text_prompt", "prompt"),+ jsonSchema: createAiImproveConfig("text_prompt", "json_schema"),+ },+ humanInteraction: {+ instructions: createAiImproveConfig("human_interaction", "instructions"),+ positiveDescriptor: createAiImproveConfig(+ "human_interaction",+ "positive_descriptor",+ ),+ negativeDescriptor: createAiImproveConfig(+ "human_interaction",+ "negative_descriptor",+ ),+ body: createAiImproveConfig("human_interaction", "body"),+ },+ sendEmail: {+ subject: createAiImproveConfig("send_email", "subject"),+ body: createAiImproveConfig("send_email", "body"),+ },+ httpRequest: {+ body: createAiImproveConfig("http_request", "body"),+ },+} as const;diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/ActionNode/ActionNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/ActionNode/ActionNode.tsxindex 64f83f5f0..7a8c0a8bd 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/ActionNode/ActionNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/ActionNode/ActionNode.tsx@@ -17,6 +17,7 @@ import { errorMappingExampleValue } from "../types";import { CodeEditor } from "@/routes/workflows/components/CodeEditor";import { Switch } from "@/components/ui/switch";import { placeholders, helpTooltips } from "../../helpContent";+import { AI_IMPROVE_CONFIGS } from "../../constants";import { WorkflowBlockInputTextarea } from "@/components/WorkflowBlockInputTextarea";import { useRerender } from "@/hooks/useRerender";import { BlockCodeEditor } from "@/routes/workflows/components/BlockCodeEditor";@@ -139,10 +140,7 @@ function ActionNode({ id, data, type }: NodeProps<ActionNode>) {<HelpTooltip content={navigationGoalTooltip} /></div><WorkflowBlockInputTextarea- aiImprove={{- context: { block_type: "Action Block" },- useCase: "task_v1_prompt",- }}+ aiImprove={AI_IMPROVE_CONFIGS.action.navigationGoal}nodeId={id}onChange={(value) => {update({ navigationGoal: value });diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/ExtractionNode/ExtractionNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/ExtractionNode/ExtractionNode.tsxindex 2e72e209e..ff7ffb1bd 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/ExtractionNode/ExtractionNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/ExtractionNode/ExtractionNode.tsx@@ -37,6 +37,7 @@ import { useUpdate } from "@/routes/workflows/editor/useUpdate";import { useRerender } from "@/hooks/useRerender";import { DisableCache } from "../DisableCache";+import { AI_IMPROVE_CONFIGS } from "../../constants";function ExtractionNode({ id, data, type }: NodeProps<ExtractionNode>) {const [facing, setFacing] = useState<"front" | "back">("front");@@ -113,6 +114,7 @@ function ExtractionNode({ id, data, type }: NodeProps<ExtractionNode>) {</div><WorkflowBlockInputTextarea+ aiImprove={AI_IMPROVE_CONFIGS.extraction.dataExtractionGoal}nodeId={id}onChange={(value) => {if (!editable) {diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/FileDownloadNode/FileDownloadNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/FileDownloadNode/FileDownloadNode.tsxindex e2cca0653..13c7d6865 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/FileDownloadNode/FileDownloadNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/FileDownloadNode/FileDownloadNode.tsx@@ -37,6 +37,7 @@ import { useRerender } from "@/hooks/useRerender";import { BROWSER_DOWNLOAD_TIMEOUT_SECONDS } from "@/api/types";import { DisableCache } from "../DisableCache";+import { AI_IMPROVE_CONFIGS } from "../../constants";const urlTooltip ="The URL Skyvern is navigating to. Leave this field blank to pick up from where the last block left off.";@@ -132,6 +133,7 @@ function FileDownloadNode({ id, data }: NodeProps<FileDownloadNode>) {<HelpTooltip content={navigationGoalTooltip} /></div><WorkflowBlockInputTextarea+ aiImprove={AI_IMPROVE_CONFIGS.fileDownload.navigationGoal}nodeId={id}onChange={(value) => {update({ navigationGoal: value });diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsxindex c3c36af82..515133d32 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx@@ -19,6 +19,7 @@ import {AccordionTrigger,} from "@/components/ui/accordion";import { useRerender } from "@/hooks/useRerender";+import { AI_IMPROVE_CONFIGS } from "../../constants";const instructionsTooltip ="Instructions shown to the user for review. Explain what needs to be reviewed and what action should be taken.";@@ -169,6 +170,7 @@ function HumanInteractionNode({<div className="space-y-2"><Label className="text-xs text-slate-300">Body</Label><WorkflowBlockInputTextarea+ aiImprove={AI_IMPROVE_CONFIGS.humanInteraction.body}nodeId={id}onChange={(value) => {update({ body: value });diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginNode.tsxindex 3f35a7394..c5a9aef67 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginNode.tsx@@ -36,6 +36,7 @@ import { useUpdate } from "@/routes/workflows/editor/useUpdate";import { useRerender } from "@/hooks/useRerender";import { DisableCache } from "../DisableCache";+import { AI_IMPROVE_CONFIGS } from "../../constants";function LoginNode({ id, data, type }: NodeProps<LoginNode>) {const blockScriptStore = useBlockScriptStore();@@ -127,6 +128,7 @@ function LoginNode({ id, data, type }: NodeProps<LoginNode>) {/></div><WorkflowBlockInputTextarea+ aiImprove={AI_IMPROVE_CONFIGS.login.navigationGoal}nodeId={id}onChange={(value) => {update({ navigationGoal: value });@@ -187,6 +189,7 @@ function LoginNode({ id, data, type }: NodeProps<LoginNode>) {Complete if...</Label><WorkflowBlockInputTextarea+ aiImprove={AI_IMPROVE_CONFIGS.login.completeCriterion}nodeId={id}onChange={(value) => {update({ completeCriterion: value });diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/NavigationNode/NavigationNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/NavigationNode/NavigationNode.tsxindex 8f148e980..339a55792 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/NavigationNode/NavigationNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/NavigationNode/NavigationNode.tsx@@ -37,6 +37,7 @@ import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuerimport { useUpdate } from "@/routes/workflows/editor/useUpdate";import { DisableCache } from "../DisableCache";+import { AI_IMPROVE_CONFIGS } from "../../constants";function NavigationNode({ id, data, type }: NodeProps<NavigationNode>) {const { blockLabel: urlBlockLabel } = useParams();@@ -136,6 +137,7 @@ function NavigationNode({ id, data, type }: NodeProps<NavigationNode>) {/></div><WorkflowBlockInputTextarea+ aiImprove={AI_IMPROVE_CONFIGS.navigation.navigationGoal}nodeId={id}onChange={(value) => {update({ navigationGoal: value });@@ -184,6 +186,9 @@ function NavigationNode({ id, data, type }: NodeProps<NavigationNode>) {Complete if...</Label><WorkflowBlockInputTextarea+ aiImprove={+ AI_IMPROVE_CONFIGS.navigation.completeCriterion+ }nodeId={id}onChange={(value) => {update({ completeCriterion: value });diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/SendEmailNode/SendEmailNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/SendEmailNode/SendEmailNode.tsxindex 644f41166..30105a998 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/SendEmailNode/SendEmailNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/SendEmailNode/SendEmailNode.tsx@@ -13,6 +13,7 @@ import { useParams } from "react-router-dom";import { statusIsRunningOrQueued } from "@/routes/tasks/types";import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";import { useUpdate } from "@/routes/workflows/editor/useUpdate";+import { AI_IMPROVE_CONFIGS } from "../../constants";function SendEmailNode({ id, data }: NodeProps<SendEmailNode>) {const { editable, label } = data;@@ -94,6 +95,7 @@ function SendEmailNode({ id, data }: NodeProps<SendEmailNode>) {<div className="space-y-2"><Label className="text-xs text-slate-300">Body</Label><WorkflowBlockInputTextarea+ aiImprove={AI_IMPROVE_CONFIGS.sendEmail.body}nodeId={id}onChange={(value) => {update({ body: value });diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/TaskNode/TaskNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/TaskNode/TaskNode.tsxindex b4855a9c0..2850f985d 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/TaskNode/TaskNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/TaskNode/TaskNode.tsx@@ -21,6 +21,7 @@ import { Handle, NodeProps, Position, useEdges, useNodes } from "@xyflow/react";import { useState } from "react";import { AppNode } from "..";import { helpTooltips, placeholders } from "../../helpContent";+import { AI_IMPROVE_CONFIGS } from "../../constants";import { getAvailableOutputParameterKeys } from "../../workflowEditorUtils";import { dataSchemaExampleValue, errorMappingExampleValue } from "../types";import { ParametersMultiSelect } from "./ParametersMultiSelect";@@ -137,10 +138,7 @@ function TaskNode({ id, data, type }: NodeProps<TaskNode>) {/></div><WorkflowBlockInputTextarea- aiImprove={{- context: { block_type: "Task Block" },- useCase: "task_v1_prompt",- }}+ aiImprove={AI_IMPROVE_CONFIGS.task.navigationGoal}nodeId={id}onChange={(value) => {update({ navigationGoal: value });@@ -176,6 +174,7 @@ function TaskNode({ id, data, type }: NodeProps<TaskNode>) {/></div><WorkflowBlockInputTextarea+ aiImprove={AI_IMPROVE_CONFIGS.task.dataExtractionGoal}nodeId={id}onChange={(value) => {update({ dataExtractionGoal: value });@@ -209,6 +208,7 @@ function TaskNode({ id, data, type }: NodeProps<TaskNode>) {Complete if...</Label><WorkflowBlockInputTextarea+ aiImprove={AI_IMPROVE_CONFIGS.task.completeCriterion}nodeId={id}onChange={(value) => {update({ completeCriterion: value });diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/Taskv2Node/Taskv2Node.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/Taskv2Node/Taskv2Node.tsxindex 55d6570c7..4ceafb499 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/Taskv2Node/Taskv2Node.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/Taskv2Node/Taskv2Node.tsx@@ -19,6 +19,7 @@ import { ModelSelector } from "@/components/ModelSelector";import { cn } from "@/util/utils";import { NodeHeader } from "../components/NodeHeader";import { NodeTabs } from "../components/NodeTabs";+import { AI_IMPROVE_CONFIGS } from "../../constants";import { useParams } from "react-router-dom";import { statusIsRunningOrQueued } from "@/routes/tasks/types";import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";@@ -107,7 +108,7 @@ function Taskv2Node({ id, data, type }: NodeProps<Taskv2Node>) {) : null}</div><WorkflowBlockInputTextarea- aiImprove={{ useCase: "task_v2_prompt" }}+ aiImprove={AI_IMPROVE_CONFIGS.taskV2.prompt}nodeId={id}onChange={(value) => {update({ prompt: value });diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptNode.tsxindex 9e15ba098..545969d37 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptNode.tsx@@ -15,6 +15,7 @@ import { useParams } from "react-router-dom";import { statusIsRunningOrQueued } from "@/routes/tasks/types";import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";import { useUpdate } from "@/routes/workflows/editor/useUpdate";+import { AI_IMPROVE_CONFIGS } from "../../constants";function TextPromptNode({ id, data }: NodeProps<TextPromptNode>) {const { editable, label } = data;@@ -75,6 +76,7 @@ function TextPromptNode({ id, data }: NodeProps<TextPromptNode>) {</div><WorkflowBlockInputTextarea+ aiImprove={AI_IMPROVE_CONFIGS.textPrompt.prompt}nodeId={id}onChange={(value) => {update({ prompt: value });diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/ValidationNode/ValidationNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/ValidationNode/ValidationNode.tsxindex dcd976875..598978660 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/ValidationNode/ValidationNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/ValidationNode/ValidationNode.tsx@@ -34,6 +34,7 @@ import { useUpdate } from "@/routes/workflows/editor/useUpdate";import { useRerender } from "@/hooks/useRerender";import { DisableCache } from "../DisableCache";+import { AI_IMPROVE_CONFIGS } from "../../constants";function ValidationNode({ id, data, type }: NodeProps<ValidationNode>) {const [facing, setFacing] = useState<"front" | "back">("front");@@ -114,6 +115,7 @@ function ValidationNode({ id, data, type }: NodeProps<ValidationNode>) {) : null}</div><WorkflowBlockInputTextarea+ aiImprove={AI_IMPROVE_CONFIGS.validation.completeCriterion}nodeId={id}onChange={(value) => {update({ completeCriterion: value });@@ -125,6 +127,7 @@ function ValidationNode({ id, data, type }: NodeProps<ValidationNode>) {<div className="space-y-2"><Label className="text-xs text-slate-300">Terminate if...</Label><WorkflowBlockInputTextarea+ aiImprove={AI_IMPROVE_CONFIGS.validation.terminateCriterion}nodeId={id}onChange={(value) => {update({ terminateCriterion: value });