Skip to content

Automate browser based workflows with AI

Magic Wand Button to improve prompts in all blocks (#4060)

Celal Zamanoglu0cbab39a2724
diff --git a/skyvern-frontend/src/routes/workflows/editor/constants.ts b/skyvern-frontend/src/routes/workflows/editor/constants.ts
index 7da2dfd33..4793dd213 100644
--- a/skyvern-frontend/src/routes/workflows/editor/constants.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/constants.ts
@@ -20,3 +20,95 @@ export const BITWARDEN_CLIENT_SECRET_AWS_SECRET_KEY =
"SKYVERN_BITWARDEN_CLIENT_SECRET";
export const BITWARDEN_MASTER_PASSWORD_AWS_SECRET_KEY =
"SKYVERN_BITWARDEN_MASTER_PASSWORD";
+
+type AiImproveConfig = {
+ useCase: string;
+ context: Record<string, unknown>;
+};
+
+const createAiImproveConfig = (
+ block: string,
+ field: string,
+ extraContext: Record<string, unknown> = {},
+): AiImproveConfig => ({
+ useCase: `workflow_editor.${block}.${field}`,
+ context: {
+ block_type: block,
+ field,
+ ...extraContext,
+ },
+});
+
+export const AI_IMPROVE_CONFIGS = {
+ task: {
+ navigationGoal: createAiImproveConfig("task", "navigation_goal"),
+ dataExtractionGoal: createAiImproveConfig("task", "data_extraction_goal"),
+ completeCriterion: createAiImproveConfig("task", "complete_criterion"),
+ },
+ action: {
+ navigationGoal: createAiImproveConfig("action", "navigation_goal"),
+ errorCodeMapping: createAiImproveConfig("action", "error_code_mapping"),
+ },
+ navigation: {
+ navigationGoal: createAiImproveConfig("navigation", "navigation_goal"),
+ completeCriterion: createAiImproveConfig(
+ "navigation",
+ "complete_criterion",
+ ),
+ },
+ extraction: {
+ dataExtractionGoal: createAiImproveConfig(
+ "extraction",
+ "data_extraction_goal",
+ ),
+ dataSchema: createAiImproveConfig("extraction", "data_schema"),
+ },
+ validation: {
+ completeCriterion: createAiImproveConfig(
+ "validation",
+ "complete_criterion",
+ ),
+ terminateCriterion: createAiImproveConfig(
+ "validation",
+ "terminate_criterion",
+ ),
+ },
+ login: {
+ navigationGoal: createAiImproveConfig("login", "navigation_goal"),
+ completeCriterion: createAiImproveConfig("login", "complete_criterion"),
+ terminateCriterion: createAiImproveConfig("login", "terminate_criterion"),
+ },
+ fileDownload: {
+ navigationGoal: createAiImproveConfig("file_download", "navigation_goal"),
+ completeCriterion: createAiImproveConfig(
+ "file_download",
+ "complete_criterion",
+ ),
+ },
+ taskV2: {
+ prompt: createAiImproveConfig("task_v2", "prompt"),
+ },
+ textPrompt: {
+ prompt: createAiImproveConfig("text_prompt", "prompt"),
+ jsonSchema: createAiImproveConfig("text_prompt", "json_schema"),
+ },
+ humanInteraction: {
+ instructions: createAiImproveConfig("human_interaction", "instructions"),
+ positiveDescriptor: createAiImproveConfig(
+ "human_interaction",
+ "positive_descriptor",
+ ),
+ negativeDescriptor: createAiImproveConfig(
+ "human_interaction",
+ "negative_descriptor",
+ ),
+ body: createAiImproveConfig("human_interaction", "body"),
+ },
+ sendEmail: {
+ subject: createAiImproveConfig("send_email", "subject"),
+ body: createAiImproveConfig("send_email", "body"),
+ },
+ httpRequest: {
+ body: createAiImproveConfig("http_request", "body"),
+ },
+} as const;
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/ActionNode/ActionNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/ActionNode/ActionNode.tsx
index 64f83f5f0..7a8c0a8bd 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/ActionNode/ActionNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/ActionNode/ActionNode.tsx
@@ -17,6 +17,7 @@ import { errorMappingExampleValue } from "../types";
import { CodeEditor } from "@/routes/workflows/components/CodeEditor";
import { Switch } from "@/components/ui/switch";
import { placeholders, helpTooltips } from "../../helpContent";
+import { AI_IMPROVE_CONFIGS } from "../../constants";
import { WorkflowBlockInputTextarea } from "@/components/WorkflowBlockInputTextarea";
import { useRerender } from "@/hooks/useRerender";
import { BlockCodeEditor } from "@/routes/workflows/components/BlockCodeEditor";
@@ -139,10 +140,7 @@ function ActionNode({ id, data, type }: NodeProps<ActionNode>) {
<HelpTooltip content={navigationGoalTooltip} />
</div>
<WorkflowBlockInputTextarea
- aiImprove={{
- context: { block_type: "Action Block" },
- useCase: "task_v1_prompt",
- }}
+ aiImprove={AI_IMPROVE_CONFIGS.action.navigationGoal}
nodeId={id}
onChange={(value) => {
update({ navigationGoal: value });
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/ExtractionNode/ExtractionNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/ExtractionNode/ExtractionNode.tsx
index 2e72e209e..ff7ffb1bd 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/ExtractionNode/ExtractionNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/ExtractionNode/ExtractionNode.tsx
@@ -37,6 +37,7 @@ import { useUpdate } from "@/routes/workflows/editor/useUpdate";
import { useRerender } from "@/hooks/useRerender";
import { DisableCache } from "../DisableCache";
+import { AI_IMPROVE_CONFIGS } from "../../constants";
function ExtractionNode({ id, data, type }: NodeProps<ExtractionNode>) {
const [facing, setFacing] = useState<"front" | "back">("front");
@@ -113,6 +114,7 @@ function ExtractionNode({ id, data, type }: NodeProps<ExtractionNode>) {
</div>
<WorkflowBlockInputTextarea
+ aiImprove={AI_IMPROVE_CONFIGS.extraction.dataExtractionGoal}
nodeId={id}
onChange={(value) => {
if (!editable) {
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/FileDownloadNode/FileDownloadNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/FileDownloadNode/FileDownloadNode.tsx
index e2cca0653..13c7d6865 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/FileDownloadNode/FileDownloadNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/FileDownloadNode/FileDownloadNode.tsx
@@ -37,6 +37,7 @@ import { useRerender } from "@/hooks/useRerender";
import { BROWSER_DOWNLOAD_TIMEOUT_SECONDS } from "@/api/types";
import { DisableCache } from "../DisableCache";
+import { AI_IMPROVE_CONFIGS } from "../../constants";
const urlTooltip =
"The URL Skyvern is navigating to. Leave this field blank to pick up from where the last block left off.";
@@ -132,6 +133,7 @@ function FileDownloadNode({ id, data }: NodeProps<FileDownloadNode>) {
<HelpTooltip content={navigationGoalTooltip} />
</div>
<WorkflowBlockInputTextarea
+ aiImprove={AI_IMPROVE_CONFIGS.fileDownload.navigationGoal}
nodeId={id}
onChange={(value) => {
update({ navigationGoal: value });
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx
index c3c36af82..515133d32 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx
@@ -19,6 +19,7 @@ import {
AccordionTrigger,
} from "@/components/ui/accordion";
import { useRerender } from "@/hooks/useRerender";
+import { AI_IMPROVE_CONFIGS } from "../../constants";
const instructionsTooltip =
"Instructions shown to the user for review. Explain what needs to be reviewed and what action should be taken.";
@@ -169,6 +170,7 @@ function HumanInteractionNode({
<div className="space-y-2">
<Label className="text-xs text-slate-300">Body</Label>
<WorkflowBlockInputTextarea
+ aiImprove={AI_IMPROVE_CONFIGS.humanInteraction.body}
nodeId={id}
onChange={(value) => {
update({ body: value });
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginNode.tsx
index 3f35a7394..c5a9aef67 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginNode.tsx
@@ -36,6 +36,7 @@ import { useUpdate } from "@/routes/workflows/editor/useUpdate";
import { useRerender } from "@/hooks/useRerender";
import { DisableCache } from "../DisableCache";
+import { AI_IMPROVE_CONFIGS } from "../../constants";
function LoginNode({ id, data, type }: NodeProps<LoginNode>) {
const blockScriptStore = useBlockScriptStore();
@@ -127,6 +128,7 @@ function LoginNode({ id, data, type }: NodeProps<LoginNode>) {
/>
</div>
<WorkflowBlockInputTextarea
+ aiImprove={AI_IMPROVE_CONFIGS.login.navigationGoal}
nodeId={id}
onChange={(value) => {
update({ navigationGoal: value });
@@ -187,6 +189,7 @@ function LoginNode({ id, data, type }: NodeProps<LoginNode>) {
Complete if...
</Label>
<WorkflowBlockInputTextarea
+ aiImprove={AI_IMPROVE_CONFIGS.login.completeCriterion}
nodeId={id}
onChange={(value) => {
update({ completeCriterion: value });
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/NavigationNode/NavigationNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/NavigationNode/NavigationNode.tsx
index 8f148e980..339a55792 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/NavigationNode/NavigationNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/NavigationNode/NavigationNode.tsx
@@ -37,6 +37,7 @@ import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuer
import { useUpdate } from "@/routes/workflows/editor/useUpdate";
import { DisableCache } from "../DisableCache";
+import { AI_IMPROVE_CONFIGS } from "../../constants";
function NavigationNode({ id, data, type }: NodeProps<NavigationNode>) {
const { blockLabel: urlBlockLabel } = useParams();
@@ -136,6 +137,7 @@ function NavigationNode({ id, data, type }: NodeProps<NavigationNode>) {
/>
</div>
<WorkflowBlockInputTextarea
+ aiImprove={AI_IMPROVE_CONFIGS.navigation.navigationGoal}
nodeId={id}
onChange={(value) => {
update({ navigationGoal: value });
@@ -184,6 +186,9 @@ function NavigationNode({ id, data, type }: NodeProps<NavigationNode>) {
Complete if...
</Label>
<WorkflowBlockInputTextarea
+ aiImprove={
+ AI_IMPROVE_CONFIGS.navigation.completeCriterion
+ }
nodeId={id}
onChange={(value) => {
update({ completeCriterion: value });
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/SendEmailNode/SendEmailNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/SendEmailNode/SendEmailNode.tsx
index 644f41166..30105a998 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/SendEmailNode/SendEmailNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/SendEmailNode/SendEmailNode.tsx
@@ -13,6 +13,7 @@ import { useParams } from "react-router-dom";
import { statusIsRunningOrQueued } from "@/routes/tasks/types";
import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";
import { useUpdate } from "@/routes/workflows/editor/useUpdate";
+import { AI_IMPROVE_CONFIGS } from "../../constants";
function SendEmailNode({ id, data }: NodeProps<SendEmailNode>) {
const { editable, label } = data;
@@ -94,6 +95,7 @@ function SendEmailNode({ id, data }: NodeProps<SendEmailNode>) {
<div className="space-y-2">
<Label className="text-xs text-slate-300">Body</Label>
<WorkflowBlockInputTextarea
+ aiImprove={AI_IMPROVE_CONFIGS.sendEmail.body}
nodeId={id}
onChange={(value) => {
update({ body: value });
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/TaskNode/TaskNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/TaskNode/TaskNode.tsx
index b4855a9c0..2850f985d 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/TaskNode/TaskNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/TaskNode/TaskNode.tsx
@@ -21,6 +21,7 @@ import { Handle, NodeProps, Position, useEdges, useNodes } from "@xyflow/react";
import { useState } from "react";
import { AppNode } from "..";
import { helpTooltips, placeholders } from "../../helpContent";
+import { AI_IMPROVE_CONFIGS } from "../../constants";
import { getAvailableOutputParameterKeys } from "../../workflowEditorUtils";
import { dataSchemaExampleValue, errorMappingExampleValue } from "../types";
import { ParametersMultiSelect } from "./ParametersMultiSelect";
@@ -137,10 +138,7 @@ function TaskNode({ id, data, type }: NodeProps<TaskNode>) {
/>
</div>
<WorkflowBlockInputTextarea
- aiImprove={{
- context: { block_type: "Task Block" },
- useCase: "task_v1_prompt",
- }}
+ aiImprove={AI_IMPROVE_CONFIGS.task.navigationGoal}
nodeId={id}
onChange={(value) => {
update({ navigationGoal: value });
@@ -176,6 +174,7 @@ function TaskNode({ id, data, type }: NodeProps<TaskNode>) {
/>
</div>
<WorkflowBlockInputTextarea
+ aiImprove={AI_IMPROVE_CONFIGS.task.dataExtractionGoal}
nodeId={id}
onChange={(value) => {
update({ dataExtractionGoal: value });
@@ -209,6 +208,7 @@ function TaskNode({ id, data, type }: NodeProps<TaskNode>) {
Complete if...
</Label>
<WorkflowBlockInputTextarea
+ aiImprove={AI_IMPROVE_CONFIGS.task.completeCriterion}
nodeId={id}
onChange={(value) => {
update({ completeCriterion: value });
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/Taskv2Node/Taskv2Node.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/Taskv2Node/Taskv2Node.tsx
index 55d6570c7..4ceafb499 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/Taskv2Node/Taskv2Node.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/Taskv2Node/Taskv2Node.tsx
@@ -19,6 +19,7 @@ import { ModelSelector } from "@/components/ModelSelector";
import { cn } from "@/util/utils";
import { NodeHeader } from "../components/NodeHeader";
import { NodeTabs } from "../components/NodeTabs";
+import { AI_IMPROVE_CONFIGS } from "../../constants";
import { useParams } from "react-router-dom";
import { statusIsRunningOrQueued } from "@/routes/tasks/types";
import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";
@@ -107,7 +108,7 @@ function Taskv2Node({ id, data, type }: NodeProps<Taskv2Node>) {
) : null}
</div>
<WorkflowBlockInputTextarea
- aiImprove={{ useCase: "task_v2_prompt" }}
+ aiImprove={AI_IMPROVE_CONFIGS.taskV2.prompt}
nodeId={id}
onChange={(value) => {
update({ prompt: value });
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptNode.tsx
index 9e15ba098..545969d37 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptNode.tsx
@@ -15,6 +15,7 @@ import { useParams } from "react-router-dom";
import { statusIsRunningOrQueued } from "@/routes/tasks/types";
import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";
import { useUpdate } from "@/routes/workflows/editor/useUpdate";
+import { AI_IMPROVE_CONFIGS } from "../../constants";
function TextPromptNode({ id, data }: NodeProps<TextPromptNode>) {
const { editable, label } = data;
@@ -75,6 +76,7 @@ function TextPromptNode({ id, data }: NodeProps<TextPromptNode>) {
</div>
<WorkflowBlockInputTextarea
+ aiImprove={AI_IMPROVE_CONFIGS.textPrompt.prompt}
nodeId={id}
onChange={(value) => {
update({ prompt: value });
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/ValidationNode/ValidationNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/ValidationNode/ValidationNode.tsx
index dcd976875..598978660 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/ValidationNode/ValidationNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/ValidationNode/ValidationNode.tsx
@@ -34,6 +34,7 @@ import { useUpdate } from "@/routes/workflows/editor/useUpdate";
import { useRerender } from "@/hooks/useRerender";
import { DisableCache } from "../DisableCache";
+import { AI_IMPROVE_CONFIGS } from "../../constants";
function ValidationNode({ id, data, type }: NodeProps<ValidationNode>) {
const [facing, setFacing] = useState<"front" | "back">("front");
@@ -114,6 +115,7 @@ function ValidationNode({ id, data, type }: NodeProps<ValidationNode>) {
) : null}
</div>
<WorkflowBlockInputTextarea
+ aiImprove={AI_IMPROVE_CONFIGS.validation.completeCriterion}
nodeId={id}
onChange={(value) => {
update({ completeCriterion: value });
@@ -125,6 +127,7 @@ function ValidationNode({ id, data, type }: NodeProps<ValidationNode>) {
<div className="space-y-2">
<Label className="text-xs text-slate-300">Terminate if...</Label>
<WorkflowBlockInputTextarea
+ aiImprove={AI_IMPROVE_CONFIGS.validation.terminateCriterion}
nodeId={id}
onChange={(value) => {
update({ terminateCriterion: value });