Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
various hitl buffs (#3828)
Jonathan Dobsonc12c047768eb
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsxindex 948894a79..2f6bfa613 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx@@ -95,94 +95,79 @@ function HumanInteractionNode({</div>{/* TODO(jdo): 'instructions' allows templating; but it requires adding a column to the workflow_block_runstable, and I don't want to do that just yet (see /timeline endpoint) */}- <Input- onChange={(event) => {- update({ instructions: event.target.value });+ <WorkflowBlockInput+ nodeId={id}+ onChange={(value) => {+ update({ instructions: value });}}value={data.instructions}placeholder="Please review and approve or reject to continue the workflow."className="nopan text-xs"/></div>- <div className="flex gap-4">- <div className="flex-1 space-y-2">- <div className="flex gap-2">- <Label className="text-xs text-slate-300">- Negative Button Label- </Label>- <HelpTooltip content={negativeDescriptorTooltip} />- </div>- <Input- onChange={(event) => {- update({ negativeDescriptor: event.target.value });- }}- value={data.negativeDescriptor}- placeholder="Reject"- className="nopan text-xs"- />- </div>- <div className="flex-1 space-y-2">- <div className="flex gap-2">- <Label className="text-xs text-slate-300">- Positive Button Label- </Label>- <HelpTooltip content={positiveDescriptorTooltip} />- </div>- <Input- onChange={(event) => {- update({ positiveDescriptor: event.target.value });- }}- value={data.positiveDescriptor}- placeholder="Approve"- className="nopan text-xs"- />- </div>- </div>- {/* <div className="space-y-2">- <div className="flex gap-2">- <Label className="text-xs text-slate-300">- Timeout (seconds)- </Label>- <HelpTooltip content={timeoutTooltip} />- </div>- <Input- type="number"- className="nopan text-xs"- min="1"- value={data.timeoutSeconds}- onChange={(event) => {- if (!editable) {- return;- }- const value = Number(event.target.value);- update({ timeoutSeconds: value });- }}- placeholder="7200"- />- </div> */}<div className="space-between flex items-center gap-2">- <Label className="text-xs text-slate-300">Timeout (seconds)</Label>+ <Label className="text-xs text-slate-300">Timeout (minutes)</Label><HelpTooltip content={timeoutTooltip} /><InputclassName="ml-auto w-16 text-right"- value={data.timeoutSeconds}- placeholder="7200"+ value={data.timeoutSeconds / 60}+ placeholder="120"onChange={(event) => {if (!editable) {return;}const value = Number(event.target.value);- update({ timeoutSeconds: value });+ update({ timeoutSeconds: value * 60 });}}/></div>- <div className="rounded-md bg-slate-800 p-2">+ <div className="flex items-center justify-center gap-2 rounded-md bg-slate-800 p-2">+ <span className="rounded bg-slate-700 p-1 text-lg">💡</span><div className="space-y-1 text-xs text-slate-400">- Tip: The workflow will pause and send an email notification to the+ The workflow will pause and send an email notification to therecipients. The workflow continues or terminates based on theuser's response.</div></div>+ <div className="space-y-4 rounded-md bg-slate-800 p-4">+ <h2>Email Settings</h2>+ <div className="space-y-2">+ <Label className="text-xs text-slate-300">Recipients</Label>+ <WorkflowBlockInput+ nodeId={id}+ onChange={(value) => {+ update({ recipients: value });+ }}+ value={data.recipients}+ placeholder="[email protected], [email protected]..."+ className="nopan text-xs"+ />+ </div>+ <div className="space-y-2">+ <Label className="text-xs text-slate-300">Subject</Label>+ <WorkflowBlockInput+ nodeId={id}+ onChange={(value) => {+ update({ subject: value });+ }}+ value={data.subject}+ placeholder="Human interaction required for workflow run"+ className="nopan text-xs"+ />+ </div>+ <div className="space-y-2">+ <Label className="text-xs text-slate-300">Body</Label>+ <WorkflowBlockInputTextarea+ nodeId={id}+ onChange={(value) => {+ update({ body: value });+ }}+ value={data.body}+ placeholder="Your interaction is required for a workflow run!"+ className="nopan text-xs"+ />+ </div>+ </div></div><Separator /><Accordion@@ -194,44 +179,46 @@ function HumanInteractionNode({collapsible><AccordionItem value="email" className="border-b-0">- <AccordionTrigger className="py-0">Email Settings</AccordionTrigger>+ <AccordionTrigger className="py-0">+ Advanced Settings+ </AccordionTrigger><AccordionContent className="pl-6 pr-1 pt-1">- <div key={rerender.key} className="space-y-4">- <div className="space-y-2">- <Label className="text-xs text-slate-300">Recipients</Label>- <WorkflowBlockInput- nodeId={id}- onChange={(value) => {- update({ recipients: value });- }}- value={data.recipients}- placeholder="[email protected], [email protected]..."- className="nopan text-xs"- />- </div>- <div className="space-y-2">- <Label className="text-xs text-slate-300">Subject</Label>- <WorkflowBlockInput- nodeId={id}- onChange={(value) => {- update({ subject: value });- }}- value={data.subject}- placeholder="Human interaction required for workflow run"- className="nopan text-xs"- />- </div>- <div className="space-y-2">- <Label className="text-xs text-slate-300">Body</Label>- <WorkflowBlockInputTextarea- nodeId={id}- onChange={(value) => {- update({ body: value });- }}- value={data.body}- placeholder="Your interaction is required for a workflow run!"- className="nopan text-xs"- />+ <div key={rerender.key} className="space-y-4 pt-4">+ <div className="flex gap-4">+ <div className="flex-1 space-y-2">+ <div className="flex gap-2">+ <Label className="text-xs text-slate-300">+ Negative Button Label+ </Label>+ <HelpTooltip content={negativeDescriptorTooltip} />+ </div>+ <WorkflowBlockInput+ nodeId={id}+ onChange={(value) => {+ update({ negativeDescriptor: value });+ }}+ value={data.negativeDescriptor}+ placeholder="Reject"+ className="nopan text-xs"+ />+ </div>+ <div className="flex-1 space-y-2">+ <div className="flex gap-2">+ <Label className="text-xs text-slate-300">+ Positive Button Label+ </Label>+ <HelpTooltip content={positiveDescriptorTooltip} />+ </div>+ <WorkflowBlockInput+ nodeId={id}+ onChange={(value) => {+ update({ positiveDescriptor: value });+ }}+ value={data.positiveDescriptor}+ placeholder="Approve"+ className="nopan text-xs"+ />+ </div></div></div></AccordionContent>diff --git a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.tsindex 3d21acb88..4f9b52b51 100644--- a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts+++ b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts@@ -2318,8 +2318,10 @@ function getWorkflowErrors(nodes: Array<AppNode>): Array<string> {});const interactionNodes = nodes.filter(isHumanInteractionNode);- interactionNodes.forEach((/* node */) => {- // pass for now+ interactionNodes.forEach((node) => {+ if (node.data.recipients.trim().length === 0) {+ errors.push(`${node.data.label}: Recipients is required.`);+ }});const navigationNodes = nodes.filter(isNavigationNode);diff --git a/skyvern-frontend/src/routes/workflows/types/workflowRunTypes.ts b/skyvern-frontend/src/routes/workflows/types/workflowRunTypes.tsindex b1df28b02..efbcfe90b 100644--- a/skyvern-frontend/src/routes/workflows/types/workflowRunTypes.ts+++ b/skyvern-frontend/src/routes/workflows/types/workflowRunTypes.ts@@ -58,6 +58,11 @@ export type WorkflowRunBlock = {// for blocks in loopcurrent_value: string | null;current_index: number | null;++ // human interaction block+ instructions?: string | null;+ positive_descriptor?: string | null;+ negative_descriptor?: string | null;};export type WorkflowRunTimelineBlockItem = {diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunHumanInteraction.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunHumanInteraction.tsxindex 105bdc469..05265b575 100644--- a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunHumanInteraction.tsx+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunHumanInteraction.tsx@@ -4,16 +4,26 @@ import { Button } from "@/components/ui/button";import { useCredentialGetter } from "@/hooks/useCredentialGetter";import { cn } from "@/util/utils";import { useMutation, useQueryClient } from "@tanstack/react-query";+import { useState } from "react";+import {+ Dialog,+ DialogClose,+ DialogContent,+ DialogDescription,+ DialogFooter,+ DialogHeader,+ DialogTitle,+} from "@/components/ui/dialog";import { toast } from "@/components/ui/use-toast";import { useWorkflowRunQuery } from "../hooks/useWorkflowRunQuery";-import { HumanInteractionBlock } from "../types/workflowTypes";+import { WorkflowRunBlock } from "../types/workflowRunTypes";interface Props {- humanInteractionBlock: HumanInteractionBlock;+ workflowRunBlock: WorkflowRunBlock;}-export function WorkflowRunHumanInteraction({ humanInteractionBlock }: Props) {+export function WorkflowRunHumanInteraction({ workflowRunBlock }: Props) {const credentialGetter = useCredentialGetter();const queryClient = useQueryClient();const { data: workflowRun } = useWorkflowRunQuery();@@ -21,11 +31,14 @@ export function WorkflowRunHumanInteraction({ humanInteractionBlock }: Props) {workflowRun && workflowRun.status === WorkflowRunStatus.Paused;const buttonLayout =- humanInteractionBlock.positive_descriptor.length < 8 &&- humanInteractionBlock.negative_descriptor.length < 8+ (workflowRunBlock.positive_descriptor?.length ?? 0) < 8 &&+ (workflowRunBlock.negative_descriptor?.length ?? 0) < 8? "inline": "stacked";+ const [isDialogOpen, setIsDialogOpen] = useState(false);+ const [choice, setChoice] = useState<"approve" | "reject" | null>(null);+const approveMutation = useMutation({mutationFn: async () => {if (!workflowRun) {@@ -45,8 +58,8 @@ export function WorkflowRunHumanInteraction({ humanInteractionBlock }: Props) {toast({variant: "success",- title: `${humanInteractionBlock.positive_descriptor}`,- description: `Successfully chose: ${humanInteractionBlock.positive_descriptor}`,+ title: `${workflowRunBlock.positive_descriptor}`,+ description: `Successfully chose: ${workflowRunBlock.positive_descriptor}`,});},onError: (error) => {@@ -77,8 +90,8 @@ export function WorkflowRunHumanInteraction({ humanInteractionBlock }: Props) {toast({variant: "success",- title: `${humanInteractionBlock.negative_descriptor}`,- description: `Successfully chose: ${humanInteractionBlock.negative_descriptor}`,+ title: `${workflowRunBlock.negative_descriptor}`,+ description: `Successfully chose: ${workflowRunBlock.negative_descriptor}`,});},onError: (error) => {@@ -96,18 +109,60 @@ export function WorkflowRunHumanInteraction({ humanInteractionBlock }: Props) {return (<div className="mt-4 flex flex-col gap-4 rounded-md bg-slate-elevation4 p-4">- <div className="text-sm">{humanInteractionBlock.instructions}</div>+ <Dialog open={isDialogOpen} onOpenChange={setIsDialogOpen}>+ <DialogContent>+ <DialogHeader>+ <DialogTitle>+ {choice === "approve"+ ? workflowRunBlock.positive_descriptor+ : workflowRunBlock.negative_descriptor}+ </DialogTitle>+ <DialogDescription>Are you sure?</DialogDescription>+ </DialogHeader>+ <DialogFooter>+ <DialogClose asChild>+ <Button variant="secondary">Back</Button>+ </DialogClose>+ <Button+ variant={choice === "reject" ? "destructive" : "default"}+ onClick={() => {+ if (choice === "approve") {+ approveMutation.mutate();+ } else if (choice === "reject") {+ rejectMutation.mutate();+ }+ }}+ >+ Proceed+ </Button>+ </DialogFooter>+ </DialogContent>+ </Dialog>++ <div className="text-sm">{workflowRunBlock.instructions}</div><divclassName={cn("flex gap-2", {"justify-between": buttonLayout === "inline","flex-col": buttonLayout === "stacked",})}>- <Button variant="destructive" onClick={() => rejectMutation.mutate()}>- <div>{humanInteractionBlock.negative_descriptor}</div>+ <Button+ variant="destructive"+ onClick={() => {+ setChoice("reject");+ setIsDialogOpen(true);+ }}+ >+ <div>{workflowRunBlock.negative_descriptor}</div></Button>- <Button variant="default" onClick={() => approveMutation.mutate()}>- <div>{humanInteractionBlock.positive_descriptor}</div>+ <Button+ variant="default"+ onClick={() => {+ setChoice("approve");+ setIsDialogOpen(true);+ }}+ >+ <div>{workflowRunBlock.positive_descriptor}</div></Button></div></div>diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunTimelineBlockItem.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunTimelineBlockItem.tsxindex 443c609df..87a72722c 100644--- a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunTimelineBlockItem.tsx+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunTimelineBlockItem.tsx@@ -5,7 +5,7 @@ import {ExternalLinkIcon,} from "@radix-ui/react-icons";import { useCallback } from "react";-import { useParams, Link } from "react-router-dom";+import { Link } from "react-router-dom";import { Status } from "@/api/types";import { formatDuration, toDuration } from "@/routes/workflows/utils";@@ -27,13 +27,8 @@ import {WorkflowRunOverviewActiveElement,} from "./WorkflowRunOverview";import { ThoughtCard } from "./ThoughtCard";-import { useWorkflowQuery } from "../hooks/useWorkflowQuery";import { ObserverThought } from "../types/workflowRunTypes";-import {- HumanInteractionBlock,- isTaskVariantBlock,- type WorkflowApiResponse,-} from "../types/workflowTypes";+import { isTaskVariantBlock } from "../types/workflowTypes";import { WorkflowRunHumanInteraction } from "./WorkflowRunHumanInteraction";type Props = {@@ -45,28 +40,6 @@ type Props = {onThoughtCardClick: (thought: ObserverThought) => void;};-const getHumanInteractionBlock = (- block: WorkflowRunBlock,- workflow?: WorkflowApiResponse,-): HumanInteractionBlock | null => {- if (block.block_type !== "human_interaction") {- return null;- }-- if (!workflow) {- return null;- }-- const blocks = workflow.workflow_definition.blocks;- const candidate = blocks.find((b) => b.label === block.label);-- if (!candidate || candidate.block_type !== "human_interaction") {- return null;- }-- return candidate as HumanInteractionBlock;-};-function WorkflowRunTimelineBlockItem({activeItem,block,@@ -75,12 +48,6 @@ function WorkflowRunTimelineBlockItem({onActionClick,onThoughtCardClick,}: Props) {- const { workflowPermanentId } = useParams();- const { data: workflow } = useWorkflowQuery({- workflowPermanentId,- });-- const humanInteractionBlock = getHumanInteractionBlock(block, workflow);const actions = block.actions ?? [];const hasActiveAction =@@ -203,10 +170,8 @@ function WorkflowRunTimelineBlockItem({) : null}</div>- {humanInteractionBlock && (- <WorkflowRunHumanInteraction- humanInteractionBlock={humanInteractionBlock}- />+ {block.block_type === "human_interaction" && (+ <WorkflowRunHumanInteraction workflowRunBlock={block} />)}{actions.map((action, index) => {