Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Add XHR.dev proxy (#6019)
Shuchang Zheng737e3c238be8
diff --git a/skyvern-frontend/src/routes/workflows/editor/debugger/getBlockParameterDependencies.test.ts b/skyvern-frontend/src/routes/workflows/editor/debugger/getBlockParameterDependencies.test.tsnew file mode 100644index 000000000..0290c5f5c--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/editor/debugger/getBlockParameterDependencies.test.ts@@ -0,0 +1,224 @@+import { describe, expect, it } from "vitest";++import { getBlockParameterDependencies } from "./getBlockParameterDependencies";+import {+ WorkflowParameterValueType,+ type WorkflowBlock,+ type WorkflowParameter,+} from "@/routes/workflows/types/workflowTypes";++function workflowParam(+ key: string,+ workflowParameterType: WorkflowParameter["workflow_parameter_type"] = "string",+): WorkflowParameter {+ return {+ parameter_type: "workflow",+ key,+ description: null,+ workflow_id: "wf-test",+ workflow_parameter_id: `wpid-${key}`,+ workflow_parameter_type: workflowParameterType,+ default_value: null,+ created_at: "",+ modified_at: "",+ deleted_at: null,+ };+}++describe("getBlockParameterDependencies", () => {+ it("returns all parameters when block is undefined", () => {+ const params = [workflowParam("a"), workflowParam("b")];+ expect(getBlockParameterDependencies(undefined, params)).toEqual(params);+ });++ it("returns all parameters when workflow has no parameters", () => {+ const block = { block_type: "navigation", label: "n" } as WorkflowBlock;+ expect(getBlockParameterDependencies(block, [])).toEqual([]);+ });++ it("collects keys from parameter_keys", () => {+ const params = [workflowParam("x"), workflowParam("y")];+ const block = {+ block_type: "navigation",+ label: "n",+ parameter_keys: ["y"],+ url: null,+ title: "",+ navigation_goal: null,+ error_code_mapping: null,+ max_retries: 0,+ max_steps_per_run: null,+ complete_on_download: false,+ download_suffix: null,+ totp_verification_url: null,+ totp_identifier: null,+ disable_cache: false,+ complete_criterion: null,+ terminate_criterion: null,+ engine: null,+ include_action_history_in_verification: false,+ } as unknown as WorkflowBlock;++ expect(getBlockParameterDependencies(block, params)).toEqual([params[1]]);+ });++ it("collects keys from jinja in string fields", () => {+ const params = [workflowParam("portal_url"), workflowParam("other")];+ const block = {+ block_type: "navigation",+ label: "n",+ parameter_keys: [],+ url: "{{ portal_url }}/login",+ title: "",+ navigation_goal: null,+ error_code_mapping: null,+ max_retries: 0,+ max_steps_per_run: null,+ complete_on_download: false,+ download_suffix: null,+ totp_verification_url: null,+ totp_identifier: null,+ disable_cache: false,+ complete_criterion: null,+ terminate_criterion: null,+ engine: null,+ include_action_history_in_verification: false,+ } as unknown as WorkflowBlock;++ expect(getBlockParameterDependencies(block, params)).toEqual([params[0]]);+ });++ it("parses jinja with filters", () => {+ const params = [workflowParam("x")];+ const block = {+ block_type: "goto_url",+ label: "u",+ url: "{{ x | default('') }}",+ } as unknown as WorkflowBlock;++ expect(getBlockParameterDependencies(block, params)).toEqual([params[0]]);+ });++ it("collects workflow params referenced inside jinja expressions", () => {+ const params = [workflowParam("count"), workflowParam("limit")];+ const block = {+ block_type: "conditional",+ label: "branch",+ branch_conditions: [+ {+ id: "1",+ criteria: {+ criteria_type: "jinja2_template",+ expression: "{{ count < limit }}",+ },+ next_block_label: null,+ description: null,+ is_default: false,+ },+ ],+ } as unknown as WorkflowBlock;++ const result = getBlockParameterDependencies(block, params);+ expect(result.map((p) => p.key).sort()).toEqual(["count", "limit"]);+ });++ it("collects workflow params from subscript jinja", () => {+ const params = [workflowParam("items"), workflowParam("other")];+ const block = {+ block_type: "goto_url",+ label: "u",+ url: "{{ items[0] }}",+ } as unknown as WorkflowBlock;++ expect(getBlockParameterDependencies(block, params)).toEqual([params[0]]);+ });++ it("resolves context parameters to their source workflow parameter", () => {+ const params = [workflowParam("source_key"), workflowParam("other")];+ const block = {+ block_type: "navigation",+ label: "n",+ parameter_keys: [],+ parameters: [+ {+ parameter_type: "context",+ key: "ctx_alias",+ description: null,+ source: workflowParam("source_key"),+ value: null,+ },+ ],+ url: null,+ title: "",+ navigation_goal: null,+ error_code_mapping: null,+ max_retries: 0,+ max_steps_per_run: null,+ complete_on_download: false,+ download_suffix: null,+ totp_verification_url: null,+ totp_identifier: null,+ disable_cache: false,+ complete_criterion: null,+ terminate_criterion: null,+ engine: null,+ include_action_history_in_verification: false,+ } as unknown as WorkflowBlock;++ expect(getBlockParameterDependencies(block, params)).toEqual([params[0]]);+ });++ it("returns empty when nothing on the block matches workflow parameters", () => {+ const params = [workflowParam("only_in_workflow")];+ const block = {+ block_type: "goto_url",+ label: "u",+ url: "https://example.com",+ } as unknown as WorkflowBlock;++ expect(getBlockParameterDependencies(block, params)).toEqual([]);+ });++ it("only includes credential params attached to the login block", () => {+ const params = [+ workflowParam("u", WorkflowParameterValueType.String),+ workflowParam("c1", WorkflowParameterValueType.CredentialId),+ workflowParam("c2", WorkflowParameterValueType.CredentialId),+ ];+ const block = {+ block_type: "login",+ label: "login",+ parameter_keys: ["u"],+ parameters: [workflowParam("c1")],+ url: null,+ title: "",+ navigation_goal: null,+ error_code_mapping: null,+ max_retries: 0,+ max_steps_per_run: null,+ totp_verification_url: null,+ totp_identifier: null,+ disable_cache: false,+ complete_criterion: null,+ terminate_criterion: null,+ engine: null,+ } as unknown as WorkflowBlock;++ const result = getBlockParameterDependencies(block, params);+ expect(result.map((p) => p.key).sort()).toEqual(["c1", "u"]);+ });++ it("includes loop_over key for for_loop blocks", () => {+ const params = [workflowParam("items"), workflowParam("other")];+ const block = {+ block_type: "for_loop",+ label: "loop",+ loop_over: workflowParam("items"),+ loop_blocks: [],+ loop_variable_reference: null,+ complete_if_empty: false,+ } as unknown as WorkflowBlock;++ expect(getBlockParameterDependencies(block, params)).toEqual([params[0]]);+ });+});diff --git a/skyvern-frontend/src/routes/workflows/editor/debugger/getBlockParameterDependencies.ts b/skyvern-frontend/src/routes/workflows/editor/debugger/getBlockParameterDependencies.tsnew file mode 100644index 000000000..18b3ac8d7--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/editor/debugger/getBlockParameterDependencies.ts@@ -0,0 +1,212 @@+import type {+ ContextParameter,+ Parameter,+ WorkflowBlock,+ WorkflowParameter,+} from "@/routes/workflows/types/workflowTypes";++const JINJA_VARIABLE_REGEX = /\{\{\s*([^}|]+?)\s*(?:\|[^}]*)?\}\}/g;+const SIMPLE_JINJA_PATH_REGEX =+ /^[a-zA-Z_][a-zA-Z0-9_]*(?:\.[a-zA-Z_][a-zA-Z0-9_]*)*$/;+const JINJA_IDENTIFIER_REGEX = /[a-zA-Z_][a-zA-Z0-9_]*/g;++const MAX_RECURSION_DEPTH = 30;++function extractJinjaRootsFromString(+ text: string,+ workflowParamKeys: Set<string>,+): string[] {+ const roots = new Set<string>();+ JINJA_VARIABLE_REGEX.lastIndex = 0;+ let match: RegExpExecArray | null;+ while ((match = JINJA_VARIABLE_REGEX.exec(text)) !== null) {+ const inner = match[1]?.trim();+ if (!inner) {+ continue;+ }++ if (SIMPLE_JINJA_PATH_REGEX.test(inner)) {+ const root = inner.split(".")[0]?.trim();+ if (root) {+ roots.add(root);+ }+ continue;+ }++ JINJA_IDENTIFIER_REGEX.lastIndex = 0;+ let identifierMatch: RegExpExecArray | null;+ while ((identifierMatch = JINJA_IDENTIFIER_REGEX.exec(inner)) !== null) {+ const identifier = identifierMatch[0];+ if (workflowParamKeys.has(identifier)) {+ roots.add(identifier);+ }+ }+ }+ return [...roots];+}++function collectTemplateStringsFromValue(+ value: unknown,+ depth: number,+ skipLoopBodies: boolean,+): string[] {+ if (depth > MAX_RECURSION_DEPTH || value === null || value === undefined) {+ return [];+ }++ if (typeof value === "string") {+ return [value];+ }++ if (Array.isArray(value)) {+ return value.flatMap((item) =>+ collectTemplateStringsFromValue(item, depth + 1, skipLoopBodies),+ );+ }++ if (typeof value !== "object") {+ return [];+ }++ const record = value as Record<string, unknown>;+ const isLoopContainer =+ record.block_type === "for_loop" || record.block_type === "while_loop";++ const strings: string[] = [];+ for (const [key, child] of Object.entries(record)) {+ if (key === "output_parameter") {+ continue;+ }+ if (key === "parameters") {+ continue;+ }+ if (skipLoopBodies && isLoopContainer && key === "loop_blocks") {+ continue;+ }+ strings.push(+ ...collectTemplateStringsFromValue(child, depth + 1, skipLoopBodies),+ );+ }+ return strings;+}++function collectReferencedKeysFromJinja(+ block: WorkflowBlock,+ workflowParamKeys: Set<string>,+): Set<string> {+ const keys = new Set<string>();+ for (const text of collectTemplateStringsFromValue(block, 0, true)) {+ for (const root of extractJinjaRootsFromString(text, workflowParamKeys)) {+ keys.add(root);+ }+ }+ return keys;+}++function readExplicitParameterKeys(block: WorkflowBlock): string[] {+ const raw = (block as { parameter_keys?: unknown }).parameter_keys;+ if (!Array.isArray(raw)) {+ return [];+ }+ return raw.filter((k): k is string => typeof k === "string" && k.length > 0);+}++function resolveWorkflowParameterKey(param: Parameter): string | null {+ if (param.parameter_type === "workflow") {+ return param.key;+ }+ if (param.parameter_type === "context") {+ return resolveWorkflowParameterKey(+ (param as ContextParameter).source as Parameter,+ );+ }+ return null;+}++function readParameterKeysFromBlockParameters(block: WorkflowBlock): string[] {+ if (!("parameters" in block) || !Array.isArray(block.parameters)) {+ return [];+ }+ const keys: string[] = [];+ for (const p of block.parameters) {+ const workflowKey = resolveWorkflowParameterKey(p);+ if (workflowKey) {+ keys.push(workflowKey);+ }+ }+ return keys;+}++function collectLoopAndBranchKeys(+ block: WorkflowBlock,+ workflowParamKeys: Set<string>,+): string[] {+ const keys: string[] = [];++ if (block.block_type === "for_loop" && block.loop_over?.key) {+ const loopKey = resolveWorkflowParameterKey(block.loop_over);+ if (loopKey) {+ keys.push(loopKey);+ }+ }++ if (block.block_type === "while_loop" && block.condition?.expression) {+ keys.push(+ ...extractJinjaRootsFromString(+ block.condition.expression,+ workflowParamKeys,+ ),+ );+ }++ if (block.block_type === "conditional") {+ for (const branch of block.branch_conditions) {+ if (branch.criteria?.expression) {+ keys.push(+ ...extractJinjaRootsFromString(+ branch.criteria.expression,+ workflowParamKeys,+ ),+ );+ }+ }+ }++ return keys;+}++function collectDependencyKeySet(+ block: WorkflowBlock,+ workflowParamKeys: Set<string>,+): Set<string> {+ const keys = new Set<string>();++ for (const k of readExplicitParameterKeys(block)) {+ keys.add(k);+ }+ for (const k of readParameterKeysFromBlockParameters(block)) {+ keys.add(k);+ }+ for (const k of collectReferencedKeysFromJinja(block, workflowParamKeys)) {+ keys.add(k);+ }+ for (const k of collectLoopAndBranchKeys(block, workflowParamKeys)) {+ keys.add(k);+ }++ return keys;+}++export function getBlockParameterDependencies(+ block: WorkflowBlock | undefined,+ workflowParameters: WorkflowParameter[],+): WorkflowParameter[] {+ if (!block || workflowParameters.length === 0) {+ return workflowParameters;+ }++ const workflowParamKeys = new Set(workflowParameters.map((p) => p.key));+ const wanted = collectDependencyKeySet(block, workflowParamKeys);++ return workflowParameters.filter((p) => wanted.has(p.key));+}diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsxindex ff5ea3b93..9ca690075 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx@@ -31,6 +31,8 @@ import {type WorkflowParameter,type Parameter,} from "@/routes/workflows/types/workflowTypes";+import { getBlockParameterDependencies } from "@/routes/workflows/editor/debugger/getBlockParameterDependencies";+import { findWorkflowBlockByLabel } from "@/routes/workflows/workflowBlockUtils";import { getInitialValues } from "@/routes/workflows/utils";import { useDebuggerLastRunValuesStore } from "@/store/DebuggerLastRunValuesStore";import { useBlockOutputStore } from "@/store/BlockOutputStore";@@ -395,7 +397,17 @@ function NodeHeader({(parameter) => parameter.parameter_type === "workflow",);- const parameters = getInitialValues(location, workflowParameters ?? []);+ const lastRunValues = workflowPermanentId+ ? useDebuggerLastRunValuesStore+ .getState()+ .getLastRunValues(workflowPermanentId)+ : null;++ const parameters = getInitialValues(+ location,+ workflowParameters ?? [],+ lastRunValues,+ );// Merge with parameter overrides if providedconst mergedParameters = opts?.parameterOverrides@@ -563,8 +575,6 @@ function NodeHeader({workflow?.workflow_definition?.parameters ?? []).filter(isWorkflowParameter);- // If there are any workflow parameters, always prompt the user- // The backend requires all params to be specified for each runif (workflowParameters.length > 0) {const lastRunValues = workflowPermanentId? useDebuggerLastRunValuesStore@@ -576,13 +586,26 @@ function NodeHeader({workflowParameters,lastRunValues,);- setCurrentParamValues(currentValues);- setParametersToPrompt(workflowParameters);- setShowParamsDialog(true);+ const block = findWorkflowBlockByLabel(blocks, blockLabel);+ const parametersToRun = getBlockParameterDependencies(+ block ?? undefined,+ workflowParameters,+ );++ if (parametersToRun.length > 0) {+ setCurrentParamValues(currentValues);+ setParametersToPrompt(parametersToRun);+ setShowParamsDialog(true);+ return;+ }++ runBlock.mutate({+ codeGen: numBlocksInWorkflow === 1,+ parameterOverrides: currentValues,+ });return;}- // No parameters, run directlyrunBlock.mutate({ codeGen: numBlocksInWorkflow === 1 });};diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.pyindex f8090e12f..44a33fc33 100644--- a/skyvern/forge/agent_functions.py+++ b/skyvern/forge/agent_functions.py@@ -673,6 +673,14 @@ class AgentFunction:"""return None+ async def wait_for_challenge_solver(self, page: Page) -> None:+ """Wait for a cloud-managed challenge solver if one is attached to the page.++ OSS no-op. Cloud overrides this so OSS-synced browser/action code can+ wait at navigation/action boundaries without importing cloud modules.+ """+ return None+async def should_shadow_extraction_cache_hit(self, task: Task) -> bool:"""Cloud-overridable sample gate for extract-information shadow mode. OSS no-op."""return Falsediff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 70a71375f..fb2e80f9c 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -864,6 +864,7 @@ class ActionHandler:page: Page,action: Action,) -> list[ActionResult]:+ await app.AGENT_FUNCTION.wait_for_challenge_solver(page=page)LOG.info("Handling action",action_type=action.action_type,@@ -894,6 +895,7 @@ class ActionHandler:handler = ActionHandler._handled_action_types[action.action_type]results = await handler(action, page, scraped_page, task, step)actions_result.extend(results)+ await app.AGENT_FUNCTION.wait_for_challenge_solver(page=page)# do the teardownteardown = ActionHandler._teardown_action_types.get(action.action_type)if teardown:diff --git a/skyvern/webeye/real_browser_state.py b/skyvern/webeye/real_browser_state.pyindex bd6462f31..8ea10ba29 100644--- a/skyvern/webeye/real_browser_state.py+++ b/skyvern/webeye/real_browser_state.py@@ -18,6 +18,7 @@ from skyvern.exceptions import (FailedToStopLoadingPage,MissingBrowserStatePage,)+from skyvern.forge import appfrom skyvern.forge.sdk.trace import tracedfrom skyvern.schemas.runs import ProxyLocationInputfrom skyvern.webeye.browser_artifacts import BrowserArtifacts, VideoArtifact@@ -149,6 +150,10 @@ class RealBrowserState(BrowserState):settle=self._wait_for_settle,wait_until=wait_until,)+ await self._wait_for_challenge_solver(page=page)++ async def _wait_for_challenge_solver(self, page: Page) -> None:+ await app.AGENT_FUNCTION.wait_for_challenge_solver(page=page)async def get_working_page(self) -> Page | None:# HACK: currently, assuming the last page is always the working page.@@ -381,6 +386,7 @@ class RealBrowserState(BrowserState):loading_time=end_time - start_time,)await self._wait_for_settle()+ await self._wait_for_challenge_solver(page=page)except Exception as e:LOG.exception(f"Error while reload url: {repr(e)}")raise FailedToReloadPage(url=page.url, error_message=repr(e))@@ -400,6 +406,10 @@ class RealBrowserState(BrowserState):wait_seconds: float = 0,must_included_tags: list[str] | None = None,) -> ScrapedPage:+ page = await self.get_working_page()+ if page is not None:+ await self._wait_for_challenge_solver(page=page)+return await scraper.scrape_website(browser_state=self,url=url,