Skip to content

Automate browser based workflows with AI

Coalesce the Validation Block and the new Human Interaction Block (#3882)

Jonathan Dobson4da7b6d4dd37
diff --git a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx
index c1c644508..cd1744f93 100644
--- a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx
@@ -60,7 +60,12 @@ import {
BITWARDEN_MASTER_PASSWORD_AWS_SECRET_KEY,
} from "./constants";
import { edgeTypes } from "./edges";
-import { AppNode, isWorkflowBlockNode, nodeTypes } from "./nodes";
+import {
+ AppNode,
+ isWorkflowBlockNode,
+ nodeTypes,
+ WorkflowBlockNode,
+} from "./nodes";
import {
ParametersState,
parameterIsSkyvernCredential,
@@ -71,6 +76,7 @@ import {
import "./reactFlowOverrideStyles.css";
import {
convertEchoParameters,
+ createNode,
descendants,
getAdditionalParametersForEmailBlock,
getOrderedChildrenBlocks,
@@ -484,6 +490,30 @@ function FlowRenderer({
doLayout(newNodesWithUpdatedParameters, newEdges);
}
+ function transmuteNode(id: string, nodeType: string) {
+ const nodeToTransmute = nodes.find((node) => node.id === id);
+
+ if (!nodeToTransmute || !isWorkflowBlockNode(nodeToTransmute)) {
+ return;
+ }
+
+ const newNode = createNode(
+ { id: nodeToTransmute.id, parentId: nodeToTransmute.parentId },
+ nodeType as NonNullable<WorkflowBlockNode["type"]>,
+ nodeToTransmute.data.label,
+ );
+
+ const newNodes = nodes.map((node) => {
+ if (node.id === id) {
+ return newNode;
+ }
+ return node;
+ });
+
+ workflowChangesStore.setHasChanges(true);
+ doLayout(newNodes, edges);
+ }
+
function toggleScript({
id,
label,
@@ -646,6 +676,8 @@ function FlowRenderer({
*/
deleteNodeCallback: (id: string) =>
setTimeout(() => deleteNode(id), 0),
+ transmuteNodeCallback: (id: string, nodeName: string) =>
+ setTimeout(() => transmuteNode(id, nodeName), 0),
toggleScriptForNodeCallback: toggleScript,
}}
>
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx
index 2f6bfa613..c3c36af82 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx
@@ -77,6 +77,17 @@ function HumanInteractionNode({
nodeId={id}
totpIdentifier={null}
totpUrl={null}
+ transmutations={{
+ blockTitle: "Validation",
+ self: "human",
+ others: [
+ {
+ label: "agent",
+ reason: "Convert to automated agent validation",
+ nodeName: "validation",
+ },
+ ],
+ }}
type={type}
/>
<div
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/ValidationNode/ValidationNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/ValidationNode/ValidationNode.tsx
index 4a0e7e4e0..aa6514f5e 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/ValidationNode/ValidationNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/ValidationNode/ValidationNode.tsx
@@ -91,6 +91,17 @@ function ValidationNode({ id, data, type }: NodeProps<ValidationNode>) {
nodeId={id}
totpIdentifier={null}
totpUrl={null}
+ transmutations={{
+ blockTitle: "Validation",
+ self: "agent",
+ others: [
+ {
+ label: "human",
+ reason: "Convert to human validation",
+ nodeName: "human_interaction",
+ },
+ ],
+ }}
type={type}
/>
<div className="space-y-2">
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/components/MicroDropdown.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/components/MicroDropdown.tsx
new file mode 100644
index 000000000..99799fa31
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/components/MicroDropdown.tsx
@@ -0,0 +1,132 @@
+import { ChevronDownIcon, ChevronUpIcon } from "@radix-ui/react-icons";
+import { useEffect, useRef, useState } from "react";
+
+import { cn } from "@/util/utils";
+
+interface Props {
+ selections: string[];
+ selected: string;
+ // --
+ onChange: (selection: string) => void;
+}
+
+function MicroDropdown({ selections, selected, onChange }: Props) {
+ const dropdownRef = useRef<HTMLDivElement>(null);
+ const [isOpen, setIsOpen] = useState(false);
+ const [openUpwards, setOpenUpwards] = useState(false);
+
+ function handleOnChange(selection: string) {
+ setIsOpen(false);
+ onChange(selection);
+ }
+
+ useEffect(() => {
+ if (!isOpen) {
+ return;
+ }
+
+ function handleClickOutside(event: MouseEvent) {
+ if (
+ dropdownRef.current &&
+ !dropdownRef.current.contains(event.target as Node)
+ ) {
+ setIsOpen(false);
+ }
+ }
+
+ document.addEventListener("mousedown", handleClickOutside, true);
+ return () => {
+ document.removeEventListener("mousedown", handleClickOutside, true);
+ };
+ }, [isOpen]);
+
+ useEffect(() => {
+ if (!isOpen) {
+ return;
+ }
+
+ function handleEscKey(event: KeyboardEvent) {
+ if (event.key === "Escape") {
+ setIsOpen(false);
+ }
+ }
+
+ document.addEventListener("keydown", handleEscKey);
+ return () => {
+ document.removeEventListener("keydown", handleEscKey);
+ };
+ }, [isOpen]);
+
+ return (
+ <div ref={dropdownRef} className="relative inline-block">
+ <div className="flex items-center gap-1">
+ <div
+ className="relative inline-flex p-0 text-xs text-slate-400"
+ onClick={() => {
+ if (!isOpen && dropdownRef.current) {
+ const rect = dropdownRef.current.getBoundingClientRect();
+ const viewportHeight = window.innerHeight;
+ const componentMiddle = rect.top + rect.height / 2;
+ setOpenUpwards(componentMiddle > viewportHeight * 0.5);
+ }
+ setIsOpen(!isOpen);
+ }}
+ >
+ [{selected}]
+ {isOpen ? (
+ <ChevronUpIcon className="ml-1 h-4 w-4" />
+ ) : (
+ <ChevronDownIcon className="ml-1 h-4 w-4" />
+ )}
+ {isOpen && (
+ <div
+ className={cn(
+ "absolute right-0 z-10 rounded-md bg-background text-xs text-slate-400",
+ "duration-200 animate-in fade-in-0 zoom-in-95",
+ openUpwards
+ ? "bottom-full mb-2 slide-in-from-bottom-2"
+ : "top-full mt-2 slide-in-from-top-2",
+ )}
+ onClick={(e) => e.stopPropagation()}
+ >
+ <div className="space-y-1 p-2">
+ {selections.map((s, index) => (
+ <div
+ key={s}
+ className={cn(
+ "flex cursor-pointer items-center gap-1 rounded-md p-1 hover:bg-slate-800",
+ "animate-in fade-in-0 slide-in-from-left-2",
+ {
+ "pointer-events-none cursor-default opacity-50":
+ s === selected,
+ },
+ )}
+ style={{
+ animationDelay: `${(index + 1) * 80}ms`,
+ animationDuration: "200ms",
+ animationFillMode: "backwards",
+ }}
+ >
+ <div
+ onClick={() => {
+ if (s === selected) {
+ return;
+ }
+
+ handleOnChange(s);
+ }}
+ >
+ {s === selected ? `[${s}]` : s}
+ </div>
+ </div>
+ ))}
+ </div>
+ </div>
+ )}
+ </div>
+ </div>
+ </div>
+ );
+}
+
+export { MicroDropdown };
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx
index 2c8b88660..da0d27062 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx
@@ -15,6 +15,7 @@ import { useAutoplayStore } from "@/store/useAutoplayStore";
import { useNodeLabelChangeHandler } from "@/routes/workflows/hooks/useLabelChangeHandler";
import { useDeleteNodeCallback } from "@/routes/workflows/hooks/useDeleteNodeCallback";
+import { useTransmuteNodeCallback } from "@/routes/workflows/hooks/useTransmuteNodeCallback";
import { useToggleScriptForNodeCallback } from "@/routes/workflows/hooks/useToggleScriptForNodeCallback";
import { useDebugSessionQuery } from "@/routes/workflows/hooks/useDebugSessionQuery";
import { useWorkflowQuery } from "@/routes/workflows/hooks/useWorkflowQuery";
@@ -45,6 +46,17 @@ import { EditableNodeTitle } from "../components/EditableNodeTitle";
import { NodeActionMenu } from "../NodeActionMenu";
import { WorkflowBlockIcon } from "../WorkflowBlockIcon";
import { workflowBlockTitle } from "../types";
+import { MicroDropdown } from "./MicroDropdown";
+
+interface Transmutations {
+ blockTitle: string;
+ self: string;
+ others: {
+ label: string;
+ reason: string;
+ nodeName: string;
+ }[];
+}
interface Props {
blockLabel: string; // today, this + wpid act as the identity of a block
@@ -53,6 +65,7 @@ interface Props {
nodeId: string;
totpIdentifier: string | null;
totpUrl: string | null;
+ transmutations?: Transmutations;
type: WorkflowBlockType;
}
@@ -144,6 +157,7 @@ function NodeHeader({
nodeId,
totpIdentifier,
totpUrl,
+ transmutations,
type,
}: Props) {
const log = useLogging();
@@ -162,6 +176,7 @@ function NodeHeader({
});
const blockTitle = workflowBlockTitle[type];
const deleteNodeCallback = useDeleteNodeCallback();
+ const transmuteNodeCallback = useTransmuteNodeCallback();
const toggleScriptForNodeCallback = useToggleScriptForNodeCallback();
const credentialGetter = useCredentialGetter();
const navigate = useNavigate();
@@ -510,7 +525,34 @@ function NodeHeader({
titleClassName="text-base"
inputClassName="text-base"
/>
- <span className="text-xs text-slate-400">{blockTitle}</span>
+
+ {transmutations && transmutations.others.length ? (
+ <div className="flex items-center gap-1">
+ <span className="text-xs text-slate-400">
+ {transmutations.blockTitle}
+ </span>
+ <MicroDropdown
+ selections={[
+ transmutations.self,
+ ...transmutations.others.map((t) => t.label),
+ ]}
+ selected={transmutations.self}
+ onChange={(label) => {
+ const transmutation = transmutations.others.find(
+ (t) => t.label === label,
+ );
+
+ if (!transmutation) {
+ return;
+ }
+
+ transmuteNodeCallback(nodeId, transmutation.nodeName);
+ }}
+ />
+ </div>
+ ) : (
+ <span className="text-xs text-slate-400">{blockTitle}</span>
+ )}
</div>
</div>
<div className="pointer-events-auto ml-auto flex items-center gap-2">
diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsx
index cdad3d2d7..ded8ab0a9 100644
--- a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsx
@@ -87,17 +87,21 @@ const nodeLibraryItems: Array<{
title: "Validation Block",
description: "Validate completion criteria",
},
- {
- nodeType: "human_interaction",
- icon: (
- <WorkflowBlockIcon
- workflowBlockType={WorkflowBlockTypes.HumanInteraction}
- className="size-6"
- />
- ),
- title: "Human Interaction Block",
- description: "Validate via human interaction",
- },
+ /**
+ * The Human Interaction block can be had via a transmutation of the
+ * Validation block.
+ */
+ // {
+ // nodeType: "human_interaction",
+ // icon: (
+ // <WorkflowBlockIcon
+ // workflowBlockType={WorkflowBlockTypes.HumanInteraction}
+ // className="size-6"
+ // />
+ // ),
+ // title: "Human Interaction Block",
+ // description: "Validate via human interaction",
+ // },
// {
// nodeType: "task",
// icon: (
diff --git a/skyvern-frontend/src/routes/workflows/editor/reactFlowOverrideStyles.css b/skyvern-frontend/src/routes/workflows/editor/reactFlowOverrideStyles.css
index fc320e638..7783f1397 100644
--- a/skyvern-frontend/src/routes/workflows/editor/reactFlowOverrideStyles.css
+++ b/skyvern-frontend/src/routes/workflows/editor/reactFlowOverrideStyles.css
@@ -14,6 +14,22 @@
@apply bg-slate-elevation3;
}
+/* Smooth transitions for node position and size changes */
+.react-flow__node {
+ transition: transform 0.3s ease-in-out;
+}
+
+/* Smooth transitions for edge position updates */
+.react-flow__edge path,
+.react-flow__edge-path {
+ transition: d 0.3s ease-in-out;
+}
+
+.react-flow__edge text,
+.react-flow__edge-text {
+ transition: transform 0.3s ease-in-out;
+}
+
.react-flow__handle-top {
top: 3px;
}
diff --git a/skyvern-frontend/src/routes/workflows/hooks/useTransmuteNodeCallback.ts b/skyvern-frontend/src/routes/workflows/hooks/useTransmuteNodeCallback.ts
new file mode 100644
index 000000000..208e356c5
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/hooks/useTransmuteNodeCallback.ts
@@ -0,0 +1,17 @@
+import { BlockActionContext } from "@/store/BlockActionContext";
+import { useContext } from "react";
+
+function useTransmuteNodeCallback() {
+ const transmuteNodeCallback =
+ useContext(BlockActionContext)?.transmuteNodeCallback;
+
+ if (!transmuteNodeCallback) {
+ throw new Error(
+ "useTransmuteNodeCallback must be used within a BlockActionContextProvider",
+ );
+ }
+
+ return transmuteNodeCallback;
+}
+
+export { useTransmuteNodeCallback };
diff --git a/skyvern-frontend/src/store/BlockActionContext.ts b/skyvern-frontend/src/store/BlockActionContext.ts
index 3a699192d..f1877466e 100644
--- a/skyvern-frontend/src/store/BlockActionContext.ts
+++ b/skyvern-frontend/src/store/BlockActionContext.ts
@@ -1,6 +1,7 @@
import { createContext } from "react";
type DeleteNodeCallback = (id: string) => void;
+type TransmuteNodeCallback = (id: string, nodeName: string) => void;
type ToggleScriptForNodeCallback = (opts: {
id?: string;
label?: string;
@@ -10,6 +11,7 @@ type ToggleScriptForNodeCallback = (opts: {
const BlockActionContext = createContext<
| {
deleteNodeCallback: DeleteNodeCallback;
+ transmuteNodeCallback: TransmuteNodeCallback;
toggleScriptForNodeCallback?: ToggleScriptForNodeCallback;
}
| undefined