Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Log cede-control with no VNC channel at info, keep take-control at warning (#8405)
Aaron Perez653ef4a1fc5b
diff --git a/skyvern-frontend/src/components/Flippable.tsx b/skyvern-frontend/src/components/Flippable.tsxindex 846db69e7..811e58068 100644--- a/skyvern-frontend/src/components/Flippable.tsx+++ b/skyvern-frontend/src/components/Flippable.tsx@@ -18,6 +18,12 @@ interface FlippableProps {preserveFrontsideHeight?: boolean;}+// The hidden face is only turned away (backfaceVisibility), so its buttons+// would stay in the tab order and the accessibility tree without inert.+// React 18 renders inert only as a string, not a boolean.+const inertWhen = (inactive: boolean): Record<string, string> =>+ inactive ? { inert: "" } : {};+export function Flippable({facing = "front",children,@@ -73,6 +79,7 @@ export function Flippable({><divref={frontRef}+ {...inertWhen(facing !== "front")}style={{backfaceVisibility: "hidden",WebkitBackfaceVisibility: "hidden",@@ -84,6 +91,7 @@ export function Flippable({</div><divref={backRef}+ {...inertWhen(facing !== "back")}className="absolute inset-0 flex items-start justify-start"style={{backfaceVisibility: "hidden",diff --git a/skyvern-frontend/src/routes/workflows/editor/WorkflowInputs.tsx b/skyvern-frontend/src/routes/workflows/editor/WorkflowInputs.tsxindex 3c6e4b49f..a8aeb76c0 100644--- a/skyvern-frontend/src/routes/workflows/editor/WorkflowInputs.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/WorkflowInputs.tsx@@ -6,9 +6,16 @@ import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";import { useWorkflowParametersStore } from "@/store/WorkflowParametersStore";import { cn } from "@/util/utils";-// A long input list would grow the start node without bound, and the canvas-// card is a summary, not the editor — the panel lists them all.-const MAX_VISIBLE_KEYS = 8;+// The node is a summary; the panel lists every input. Six names fill about+// two lines of the 30rem card. Keys have no length limit, so each is cut to+// what the panel's 12rem name column shows; the full sentence stays in the+// accessible name and the tooltip.+const MAX_VISIBLE_KEYS = 6;+const MAX_KEY_CHARS = 24;++// The editor's hand-rolled controls all carry this ring.+const FOCUS_RING =+ "rounded-md focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50";/*** Count chip for the editor headers' Inputs control. `Badge` is px-2.5 py-1,@@ -38,54 +45,93 @@ function InputsCountBadge({);}+function summarize(keys: string[], maxKeyChars = Infinity): string {+ const hidden = keys.length - MAX_VISIBLE_KEYS;+ const shown = keys+ .slice(0, MAX_VISIBLE_KEYS)+ .map((key) =>+ key.length > maxKeyChars ? `${key.slice(0, maxKeyChars - 1)}\u2026` : key,+ )+ .join(", ");+ return hidden > 0 ? `${shown}, and ${hidden} more` : shown;+}+/*** Inputs, stated where the flow starts, because a header toggle was the only- * door and nobody found it (SKY-14866). Its own card rather than a row in the- * start node's settings accordion, which already carries thirteen configs.+ * door and nobody found it (SKY-14866). A section of the Start card under its+ * heading, beside Workflow Settings rather than inside it: that accordion+ * already carries thirteen configs and would hide Inputs again.*- * A summary, not an editor: the keys are text, and every action routes to the- * existing Inputs panel.- *- * `editable` is the start node's own flag (false for global workflows and- * deleted snapshots, matching what the editor headers hide). The summary still- * renders when it is false — reading a view-only agent's inputs is fine — but- * Add is gone, because WorkflowParametersPanel can mutate and would dirty a- * workflow that cannot be saved.+ * A summary in plain language, not an editor: the names read as a sentence in+ * the body font, and the heading, the sentence and Add all open the existing+ * Inputs panel. The heading row copies the sibling accordion trigger's grammar+ * (h3 wrapping a button, label underlines on hover) so the two rows behave+ * alike. `editable` is the start node's own flag (false for global workflows+ * and deleted snapshots, matching what the editor headers hide). The summary+ * still renders when it is false, but nothing is clickable, because+ * WorkflowParametersPanel can mutate and would dirty a workflow that cannot be+ * saved.*/-function WorkflowInputsCard({ editable }: { editable: boolean }) {+function WorkflowInputsSection({ editable }: { editable: boolean }) {const parameters = useWorkflowParametersStore((s) => s.parameters);const isRecording = useRecordingStore((s) => s.isRecording);const setWorkflowPanelState = useWorkflowPanelStore((s) => s.setWorkflowPanelState,);- const hidden = parameters.length - MAX_VISIBLE_KEYS;+ const openPanel = () =>+ setWorkflowPanelState({ active: true, content: "parameters" });// Nothing to summarise and nothing to offer.if (!editable && parameters.length === 0) {return null;}+ const heading = (+ <>+ <InputIcon className="size-4 text-muted-foreground" aria-hidden />+ <span className={cn({ "group-hover:underline": editable })}>Inputs</span>+ <InputsCountBadge count={parameters.length} />+ </>+ );+ const keys = parameters.map((parameter) => parameter.key);+ const summary = summarize(keys, MAX_KEY_CHARS);+ const fullSummary = summarize(keys);+return (- // The card sits inside the start node, whose canvas click selects it and- // expands Workflow Settings (FlowRenderer's onNodeClick). This summary is- // its own surface, so no click in it reaches that.+ // Clicks stop here: a canvas click on the start node expands Workflow+ // Settings (FlowRenderer's onNodeClick), and reading a name must not. pl-6+ // aligns with the settings trigger's label, which sits after its chevron.<div- className="nodrag nopan w-[30rem] rounded-lg bg-slate-elevation3 px-6 py-4"+ className="nodrag nopan mt-3 pl-6 text-left"onClick={(event) => event.stopPropagation()}><div className="flex items-center justify-between gap-3">- <span className="flex items-center gap-2 text-sm">- <InputIcon className="size-4 text-muted-foreground" aria-hidden />- Inputs- </span>+ <h3 className="flex min-w-0">+ {editable ? (+ <button+ type="button"+ disabled={isRecording}+ onClick={openPanel}+ className={cn(+ "group flex items-center gap-2 py-1 text-sm font-medium",+ FOCUS_RING,+ )}+ >+ {heading}+ </button>+ ) : (+ <span className="flex items-center gap-2 py-1 text-sm font-medium">+ {heading}+ </span>+ )}+ </h3>{editable ? (<Button- variant="tertiary"+ variant="ghost"size="sm"+ className="-mr-3 shrink-0 text-muted-foreground hover:text-foreground"disabled={isRecording}- onClick={() =>- setWorkflowPanelState({ active: true, content: "parameters" })- }+ onClick={openPanel}><PlusIcon className="mr-1 size-3.5" aria-hidden />Add@@ -93,30 +139,33 @@ function WorkflowInputsCard({ editable }: { editable: boolean }) {) : null}</div>{parameters.length === 0 ? (- <p className="mt-2 text-left text-xs text-muted-foreground">- None yet. Inputs are placeholder values you can link in any block, and- fill in before each run.+ <p className="pl-6 text-sm text-muted-foreground">+ None yet. Add placeholders to fill in before each run.</p>+ ) : editable ? (+ <button+ type="button"+ disabled={isRecording}+ onClick={openPanel}+ title={fullSummary}+ aria-label={fullSummary}+ className={cn(+ "ml-6 block break-words text-left text-sm hover:underline",+ FOCUS_RING,+ )}+ >+ {summary}+ </button>) : (- <div className="mt-3 flex flex-wrap gap-2">- {parameters.slice(0, MAX_VISIBLE_KEYS).map((parameter) => (- <span- key={parameter.key}- title={parameter.key}- className="max-w-[10rem] truncate rounded-md bg-slate-elevation1 px-2 py-1 font-mono text-xs"- >- {parameter.key}- </span>- ))}- {hidden > 0 ? (- <span className="px-1 py-1 text-xs text-muted-foreground">- +{hidden} more- </span>- ) : null}- </div>+ // aria-label is not allowed on a paragraph, so the full text rides+ // along visually hidden instead.+ <p className="break-words pl-6 text-sm" title={fullSummary}>+ <span aria-hidden>{summary}</span>+ <span className="sr-only">{fullSummary}</span>+ </p>)}</div>);}-export { InputsCountBadge, WorkflowInputsCard };+export { InputsCountBadge, WorkflowInputsSection };diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/StartNode.test.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/StartNode.test.tsxindex 7f14e4bf9..765d5db90 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/StartNode.test.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/StartNode.test.tsx@@ -6,6 +6,7 @@ import {fireEvent,render,screen,+ within,} from "@testing-library/react";import { MemoryRouter } from "react-router-dom";import { afterEach, describe, expect, test, vi } from "vitest";@@ -99,7 +100,42 @@ afterEach(() => {});describe("StartNode inputs summary", () => {- test("states the declared inputs, and Add opens the Inputs panel", () => {+ test("states the declared inputs as a sentence, and Add opens the Inputs panel", () => {+ // Seven names: six read out, the rest counted, in the body font.+ const keys = [+ "order_id",+ "vendor_email",+ "invoice_total",+ "due_date",+ "po_number",+ "cost_center",+ "currency",+ ];+ useWorkflowParametersStore.setState({+ parameters: keys.map((key) => ({+ key,+ parameterType: "workflow",+ dataType: "string",+ defaultValue: null,+ })),+ });+ renderStartNode();++ expect(+ screen.getByText(+ "order_id, vendor_email, invoice_total, due_date, po_number, cost_center, and 1 more",+ ),+ ).toBeDefined();++ fireEvent.click(screen.getByRole("button", { name: /add/i }));++ expect(useWorkflowPanelStore.getState().workflowPanelState).toEqual({+ active: true,+ content: "parameters",+ });+ });++ test("the Inputs heading and the names each open the Inputs panel on their own", () => {useWorkflowParametersStore.setState({parameters: [{@@ -108,8 +144,46 @@ describe("StartNode inputs summary", () => {dataType: "string",defaultValue: null,},+ ],+ });+ renderStartNode();++ fireEvent.click(screen.getByRole("button", { name: "Inputs" }));+ expect(useWorkflowPanelStore.getState().workflowPanelState.active).toBe(+ true,+ );++ useWorkflowPanelStore.setState({+ workflowPanelState: { active: false, content: "parameters" },+ });+ fireEvent.click(screen.getByRole("button", { name: "order_id" }));+ expect(useWorkflowPanelStore.getState().workflowPanelState).toEqual({+ active: true,+ content: "parameters",+ });+ });++ test("the flipped code view takes the Inputs controls out of reach", () => {+ // Flippable only turns the front face away, so without inert a keyboard+ // user could tab to the invisible Add and open the panel over the script.+ // Exactly one face is inert either way, so the visible one never is.+ const flipped = renderStartNode({ showCode: true });+ expect(screen.getByText("Add").closest("[inert]")).not.toBeNull();+ expect(flipped.container.querySelectorAll("[inert]")).toHaveLength(1);++ cleanup();+ const front = renderStartNode();+ expect(screen.getByText("Add").closest("[inert]")).toBeNull();+ expect(front.container.querySelectorAll("[inert]")).toHaveLength(1);+ });++ test("a long input name is cut in the sentence but kept for assistive tech", () => {+ // Keys have no length limit; one long key must not grow the Start card.+ const key = "shipping_address_line_two_for_the_billing_contact";+ useWorkflowParametersStore.setState({+ parameters: [{- key: "vendor_email",+ key,parameterType: "workflow",dataType: "string",defaultValue: null,@@ -118,15 +192,21 @@ describe("StartNode inputs summary", () => {});renderStartNode();- expect(screen.getByText("order_id")).toBeDefined();- expect(screen.getByText("vendor_email")).toBeDefined();+ const sentence = screen.getByRole("button", { name: key });+ expect(sentence.textContent).toBe("shipping_address_line_t\u2026");+ expect(sentence.getAttribute("title")).toBe(key);- fireEvent.click(screen.getByRole("button", { name: /add/i }));-- expect(useWorkflowPanelStore.getState().workflowPanelState).toEqual({- active: true,- content: "parameters",- });+ // View-only has no button to carry an aria-label, so the full key rides+ // along visually hidden and the cut text is hidden from assistive tech.+ cleanup();+ renderStartNode({ editable: false });+ const paragraph = screen.getByTitle(key);+ expect(within(paragraph).getByText(key).className).toContain("sr-only");+ expect(+ within(paragraph)+ .getByText("shipping_address_line_t\u2026")+ .getAttribute("aria-hidden"),+ ).toBe("true");});test("an agent with no inputs says what inputs are, rather than nothing", () => {@@ -137,6 +217,29 @@ describe("StartNode inputs summary", () => {expect(screen.getByText(/None yet/)).toBeDefined();});+ test("Inputs sit under the Start heading and above Workflow Settings", () => {+ // Start is the first thing on the canvas; Inputs read as something Start+ // declares, next to Workflow Settings (SKY-15467).+ useWorkflowParametersStore.setState({+ parameters: [+ {+ key: "order_id",+ parameterType: "workflow",+ dataType: "string",+ defaultValue: null,+ },+ ],+ });+ renderStartNode();++ const precedes = (a: Element, b: Element) =>+ Boolean(a.compareDocumentPosition(b) & Node.DOCUMENT_POSITION_FOLLOWING);+ const start = screen.getByText("Start");+ const inputs = screen.getByText("Inputs");+ expect(precedes(start, inputs)).toBe(true);+ expect(precedes(inputs, screen.getByText("Workflow Settings"))).toBe(true);+ });+test("a view-only workflow can read its inputs but not add one", () => {// editable is false for global workflows and deleted snapshots, which the// editor headers already hide Inputs for. WorkflowParametersPanel mutates,@@ -153,7 +256,9 @@ describe("StartNode inputs summary", () => {});renderStartNode({ editable: false });- expect(screen.getByText("order_id")).toBeDefined();+ expect(screen.getByTitle("order_id").textContent).toContain("order_id");+ expect(screen.queryByRole("button", { name: "Inputs" })).toBeNull();+ expect(screen.queryByRole("button", { name: "order_id" })).toBeNull();expect(screen.queryByRole("button", { name: /add/i })).toBeNull();});diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/StartNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/StartNode.tsxindex 1ae048abb..aad06c43e 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/StartNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/StartNode.tsx@@ -25,7 +25,7 @@ import { BlockCodeEditor } from "@/routes/workflows/components/BlockCodeEditor";import { cn } from "@/util/utils";import { BuildModeOnly } from "../BuildModeOnly";import { isLoopNode } from "../LoopNode/types";-import { WorkflowInputsCard } from "../../WorkflowInputs";+import { WorkflowInputsSection } from "../../WorkflowInputs";import { WorkflowSettingsEditor } from "./WorkflowSettingsEditor";interface StartSettings {@@ -166,72 +166,68 @@ function StartNode({ id, data, parentId }: NodeProps<StartNode>) {if (data.withWorkflowSettings) {return (- <div className="space-y-3">- <BuildModeOnly renderInReadOnlyComparison={false}>- <WorkflowInputsCard editable={data.editable} />- </BuildModeOnly>- <Flippable facing={facing} preserveFrontsideHeight={true}>- <div>- <Handle- type="source"- position={Position.Bottom}- id="a"- className="opacity-0"- />- <div- className={cn(- "w-[30rem] rounded-lg bg-slate-elevation3 px-6 py-4 text-center",- { "h-[20rem] overflow-hidden": facing === "back" },- )}- >- <div className="relative">- <header className="mb-6 mt-2">Start</header>- <Separator />- <BuildModeOnly renderInReadOnlyComparison={false}>- <Accordion- type="single"- collapsible- value={settingsAccordionValue}- onValueChange={(value) => {- setSettingsAccordionValue(value);- rerender.bump();- }}- >- <AccordionItem value="settings" className="mt-4 border-b-0">- <AccordionTrigger className="py-2">- {/* Wrapped so the open-state chevron rotation targets+ <Flippable facing={facing} preserveFrontsideHeight={true}>+ <div>+ <Handle+ type="source"+ position={Position.Bottom}+ id="a"+ className="opacity-0"+ />+ <div+ className={cn(+ "w-[30rem] rounded-lg bg-slate-elevation3 px-6 py-4 text-center",+ { "h-[20rem] overflow-hidden": facing === "back" },+ )}+ >+ <div className="relative">+ <header className="mb-4 mt-2">Start</header>+ <Separator />+ <BuildModeOnly renderInReadOnlyComparison={false}>+ <WorkflowInputsSection editable={data.editable} />+ <Accordion+ type="single"+ collapsible+ value={settingsAccordionValue}+ onValueChange={(value) => {+ setSettingsAccordionValue(value);+ rerender.bump();+ }}+ >+ <AccordionItem value="settings" className="mt-1 border-b-0">+ <AccordionTrigger className="py-2">+ {/* Wrapped so the open-state chevron rotation targetsonly the trigger's own direct svg. */}- <span className="flex items-center gap-2">- <GearIcon- className="h-4 w-4 text-muted-foreground"- aria-hidden- />- Workflow Settings- </span>- </AccordionTrigger>- <AccordionContent className="pl-6 pr-1 pt-1">- <div key={rerender.key}>- <WorkflowSettingsEditor blockId={id} />- </div>- </AccordionContent>- </AccordionItem>- </Accordion>- </BuildModeOnly>- </div>+ <span className="flex items-center gap-2">+ <GearIcon+ className="h-4 w-4 text-muted-foreground"+ aria-hidden+ />+ Workflow Settings+ </span>+ </AccordionTrigger>+ <AccordionContent className="pl-6 pr-1 pt-1">+ <div key={rerender.key}>+ <WorkflowSettingsEditor blockId={id} />+ </div>+ </AccordionContent>+ </AccordionItem>+ </Accordion>+ </BuildModeOnly></div></div>+ </div>- <BlockCodeEditor- blockLabel="__start_block__"- title="Start"- script={script}- onExit={() => {- hideAllScripts();- return false;- }}- />- </Flippable>- </div>+ <BlockCodeEditor+ blockLabel="__start_block__"+ title="Start"+ script={script}+ onExit={() => {+ hideAllScripts();+ return false;+ }}+ />+ </Flippable>);}diff --git a/skyvern/forge/sdk/routes/streaming/channels/message.py b/skyvern/forge/sdk/routes/streaming/channels/message.pyindex 9e67a4721..994464a30 100644--- a/skyvern/forge/sdk/routes/streaming/channels/message.py+++ b/skyvern/forge/sdk/routes/streaming/channels/message.py@@ -739,7 +739,7 @@ async def loop_stream_messages(message_channel: MessageChannel) -> None:vnc_channel = get_vnc_channel(message_channel.client_id)if not vnc_channel:- LOG.warning(+ LOG.info(f"{class_name} no vnc channel client found for message channel.",message=message,**message_channel.identity,