Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Coalesce the Validation Block and the new Human Interaction Block (#3882)
Jonathan Dobson4da7b6d4dd37
diff --git a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsxindex c1c644508..cd1744f93 100644--- a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx@@ -60,7 +60,12 @@ import {BITWARDEN_MASTER_PASSWORD_AWS_SECRET_KEY,} from "./constants";import { edgeTypes } from "./edges";-import { AppNode, isWorkflowBlockNode, nodeTypes } from "./nodes";+import {+ AppNode,+ isWorkflowBlockNode,+ nodeTypes,+ WorkflowBlockNode,+} from "./nodes";import {ParametersState,parameterIsSkyvernCredential,@@ -71,6 +76,7 @@ import {import "./reactFlowOverrideStyles.css";import {convertEchoParameters,+ createNode,descendants,getAdditionalParametersForEmailBlock,getOrderedChildrenBlocks,@@ -484,6 +490,30 @@ function FlowRenderer({doLayout(newNodesWithUpdatedParameters, newEdges);}+ function transmuteNode(id: string, nodeType: string) {+ const nodeToTransmute = nodes.find((node) => node.id === id);++ if (!nodeToTransmute || !isWorkflowBlockNode(nodeToTransmute)) {+ return;+ }++ const newNode = createNode(+ { id: nodeToTransmute.id, parentId: nodeToTransmute.parentId },+ nodeType as NonNullable<WorkflowBlockNode["type"]>,+ nodeToTransmute.data.label,+ );++ const newNodes = nodes.map((node) => {+ if (node.id === id) {+ return newNode;+ }+ return node;+ });++ workflowChangesStore.setHasChanges(true);+ doLayout(newNodes, edges);+ }+function toggleScript({id,label,@@ -646,6 +676,8 @@ function FlowRenderer({*/deleteNodeCallback: (id: string) =>setTimeout(() => deleteNode(id), 0),+ transmuteNodeCallback: (id: string, nodeName: string) =>+ setTimeout(() => transmuteNode(id, nodeName), 0),toggleScriptForNodeCallback: toggleScript,}}>diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsxindex 2f6bfa613..c3c36af82 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx@@ -77,6 +77,17 @@ function HumanInteractionNode({nodeId={id}totpIdentifier={null}totpUrl={null}+ transmutations={{+ blockTitle: "Validation",+ self: "human",+ others: [+ {+ label: "agent",+ reason: "Convert to automated agent validation",+ nodeName: "validation",+ },+ ],+ }}type={type}/><divdiff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/ValidationNode/ValidationNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/ValidationNode/ValidationNode.tsxindex 4a0e7e4e0..aa6514f5e 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/ValidationNode/ValidationNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/ValidationNode/ValidationNode.tsx@@ -91,6 +91,17 @@ function ValidationNode({ id, data, type }: NodeProps<ValidationNode>) {nodeId={id}totpIdentifier={null}totpUrl={null}+ transmutations={{+ blockTitle: "Validation",+ self: "agent",+ others: [+ {+ label: "human",+ reason: "Convert to human validation",+ nodeName: "human_interaction",+ },+ ],+ }}type={type}/><div className="space-y-2">diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/components/MicroDropdown.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/components/MicroDropdown.tsxnew file mode 100644index 000000000..99799fa31--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/components/MicroDropdown.tsx@@ -0,0 +1,132 @@+import { ChevronDownIcon, ChevronUpIcon } from "@radix-ui/react-icons";+import { useEffect, useRef, useState } from "react";++import { cn } from "@/util/utils";++interface Props {+ selections: string[];+ selected: string;+ // --+ onChange: (selection: string) => void;+}++function MicroDropdown({ selections, selected, onChange }: Props) {+ const dropdownRef = useRef<HTMLDivElement>(null);+ const [isOpen, setIsOpen] = useState(false);+ const [openUpwards, setOpenUpwards] = useState(false);++ function handleOnChange(selection: string) {+ setIsOpen(false);+ onChange(selection);+ }++ useEffect(() => {+ if (!isOpen) {+ return;+ }++ function handleClickOutside(event: MouseEvent) {+ if (+ dropdownRef.current &&+ !dropdownRef.current.contains(event.target as Node)+ ) {+ setIsOpen(false);+ }+ }++ document.addEventListener("mousedown", handleClickOutside, true);+ return () => {+ document.removeEventListener("mousedown", handleClickOutside, true);+ };+ }, [isOpen]);++ useEffect(() => {+ if (!isOpen) {+ return;+ }++ function handleEscKey(event: KeyboardEvent) {+ if (event.key === "Escape") {+ setIsOpen(false);+ }+ }++ document.addEventListener("keydown", handleEscKey);+ return () => {+ document.removeEventListener("keydown", handleEscKey);+ };+ }, [isOpen]);++ return (+ <div ref={dropdownRef} className="relative inline-block">+ <div className="flex items-center gap-1">+ <div+ className="relative inline-flex p-0 text-xs text-slate-400"+ onClick={() => {+ if (!isOpen && dropdownRef.current) {+ const rect = dropdownRef.current.getBoundingClientRect();+ const viewportHeight = window.innerHeight;+ const componentMiddle = rect.top + rect.height / 2;+ setOpenUpwards(componentMiddle > viewportHeight * 0.5);+ }+ setIsOpen(!isOpen);+ }}+ >+ [{selected}]+ {isOpen ? (+ <ChevronUpIcon className="ml-1 h-4 w-4" />+ ) : (+ <ChevronDownIcon className="ml-1 h-4 w-4" />+ )}+ {isOpen && (+ <div+ className={cn(+ "absolute right-0 z-10 rounded-md bg-background text-xs text-slate-400",+ "duration-200 animate-in fade-in-0 zoom-in-95",+ openUpwards+ ? "bottom-full mb-2 slide-in-from-bottom-2"+ : "top-full mt-2 slide-in-from-top-2",+ )}+ onClick={(e) => e.stopPropagation()}+ >+ <div className="space-y-1 p-2">+ {selections.map((s, index) => (+ <div+ key={s}+ className={cn(+ "flex cursor-pointer items-center gap-1 rounded-md p-1 hover:bg-slate-800",+ "animate-in fade-in-0 slide-in-from-left-2",+ {+ "pointer-events-none cursor-default opacity-50":+ s === selected,+ },+ )}+ style={{+ animationDelay: `${(index + 1) * 80}ms`,+ animationDuration: "200ms",+ animationFillMode: "backwards",+ }}+ >+ <div+ onClick={() => {+ if (s === selected) {+ return;+ }++ handleOnChange(s);+ }}+ >+ {s === selected ? `[${s}]` : s}+ </div>+ </div>+ ))}+ </div>+ </div>+ )}+ </div>+ </div>+ </div>+ );+}++export { MicroDropdown };diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsxindex 2c8b88660..da0d27062 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx@@ -15,6 +15,7 @@ import { useAutoplayStore } from "@/store/useAutoplayStore";import { useNodeLabelChangeHandler } from "@/routes/workflows/hooks/useLabelChangeHandler";import { useDeleteNodeCallback } from "@/routes/workflows/hooks/useDeleteNodeCallback";+import { useTransmuteNodeCallback } from "@/routes/workflows/hooks/useTransmuteNodeCallback";import { useToggleScriptForNodeCallback } from "@/routes/workflows/hooks/useToggleScriptForNodeCallback";import { useDebugSessionQuery } from "@/routes/workflows/hooks/useDebugSessionQuery";import { useWorkflowQuery } from "@/routes/workflows/hooks/useWorkflowQuery";@@ -45,6 +46,17 @@ import { EditableNodeTitle } from "../components/EditableNodeTitle";import { NodeActionMenu } from "../NodeActionMenu";import { WorkflowBlockIcon } from "../WorkflowBlockIcon";import { workflowBlockTitle } from "../types";+import { MicroDropdown } from "./MicroDropdown";++interface Transmutations {+ blockTitle: string;+ self: string;+ others: {+ label: string;+ reason: string;+ nodeName: string;+ }[];+}interface Props {blockLabel: string; // today, this + wpid act as the identity of a block@@ -53,6 +65,7 @@ interface Props {nodeId: string;totpIdentifier: string | null;totpUrl: string | null;+ transmutations?: Transmutations;type: WorkflowBlockType;}@@ -144,6 +157,7 @@ function NodeHeader({nodeId,totpIdentifier,totpUrl,+ transmutations,type,}: Props) {const log = useLogging();@@ -162,6 +176,7 @@ function NodeHeader({});const blockTitle = workflowBlockTitle[type];const deleteNodeCallback = useDeleteNodeCallback();+ const transmuteNodeCallback = useTransmuteNodeCallback();const toggleScriptForNodeCallback = useToggleScriptForNodeCallback();const credentialGetter = useCredentialGetter();const navigate = useNavigate();@@ -510,7 +525,34 @@ function NodeHeader({titleClassName="text-base"inputClassName="text-base"/>- <span className="text-xs text-slate-400">{blockTitle}</span>++ {transmutations && transmutations.others.length ? (+ <div className="flex items-center gap-1">+ <span className="text-xs text-slate-400">+ {transmutations.blockTitle}+ </span>+ <MicroDropdown+ selections={[+ transmutations.self,+ ...transmutations.others.map((t) => t.label),+ ]}+ selected={transmutations.self}+ onChange={(label) => {+ const transmutation = transmutations.others.find(+ (t) => t.label === label,+ );++ if (!transmutation) {+ return;+ }++ transmuteNodeCallback(nodeId, transmutation.nodeName);+ }}+ />+ </div>+ ) : (+ <span className="text-xs text-slate-400">{blockTitle}</span>+ )}</div></div><div className="pointer-events-auto ml-auto flex items-center gap-2">diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsxindex cdad3d2d7..ded8ab0a9 100644--- a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsx@@ -87,17 +87,21 @@ const nodeLibraryItems: Array<{title: "Validation Block",description: "Validate completion criteria",},- {- nodeType: "human_interaction",- icon: (- <WorkflowBlockIcon- workflowBlockType={WorkflowBlockTypes.HumanInteraction}- className="size-6"- />- ),- title: "Human Interaction Block",- description: "Validate via human interaction",- },+ /**+ * The Human Interaction block can be had via a transmutation of the+ * Validation block.+ */+ // {+ // nodeType: "human_interaction",+ // icon: (+ // <WorkflowBlockIcon+ // workflowBlockType={WorkflowBlockTypes.HumanInteraction}+ // className="size-6"+ // />+ // ),+ // title: "Human Interaction Block",+ // description: "Validate via human interaction",+ // },// {// nodeType: "task",// icon: (diff --git a/skyvern-frontend/src/routes/workflows/editor/reactFlowOverrideStyles.css b/skyvern-frontend/src/routes/workflows/editor/reactFlowOverrideStyles.cssindex fc320e638..7783f1397 100644--- a/skyvern-frontend/src/routes/workflows/editor/reactFlowOverrideStyles.css+++ b/skyvern-frontend/src/routes/workflows/editor/reactFlowOverrideStyles.css@@ -14,6 +14,22 @@@apply bg-slate-elevation3;}+/* Smooth transitions for node position and size changes */+.react-flow__node {+ transition: transform 0.3s ease-in-out;+}++/* Smooth transitions for edge position updates */+.react-flow__edge path,+.react-flow__edge-path {+ transition: d 0.3s ease-in-out;+}++.react-flow__edge text,+.react-flow__edge-text {+ transition: transform 0.3s ease-in-out;+}+.react-flow__handle-top {top: 3px;}diff --git a/skyvern-frontend/src/routes/workflows/hooks/useTransmuteNodeCallback.ts b/skyvern-frontend/src/routes/workflows/hooks/useTransmuteNodeCallback.tsnew file mode 100644index 000000000..208e356c5--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/hooks/useTransmuteNodeCallback.ts@@ -0,0 +1,17 @@+import { BlockActionContext } from "@/store/BlockActionContext";+import { useContext } from "react";++function useTransmuteNodeCallback() {+ const transmuteNodeCallback =+ useContext(BlockActionContext)?.transmuteNodeCallback;++ if (!transmuteNodeCallback) {+ throw new Error(+ "useTransmuteNodeCallback must be used within a BlockActionContextProvider",+ );+ }++ return transmuteNodeCallback;+}++export { useTransmuteNodeCallback };diff --git a/skyvern-frontend/src/store/BlockActionContext.ts b/skyvern-frontend/src/store/BlockActionContext.tsindex 3a699192d..f1877466e 100644--- a/skyvern-frontend/src/store/BlockActionContext.ts+++ b/skyvern-frontend/src/store/BlockActionContext.ts@@ -1,6 +1,7 @@import { createContext } from "react";type DeleteNodeCallback = (id: string) => void;+type TransmuteNodeCallback = (id: string, nodeName: string) => void;type ToggleScriptForNodeCallback = (opts: {id?: string;label?: string;@@ -10,6 +11,7 @@ type ToggleScriptForNodeCallback = (opts: {const BlockActionContext = createContext<| {deleteNodeCallback: DeleteNodeCallback;+ transmuteNodeCallback: TransmuteNodeCallback;toggleScriptForNodeCallback?: ToggleScriptForNodeCallback;}| undefined