Skip to content

Automate browser based workflows with AI

Add XHR.dev proxy (#6019)

Shuchang Zheng737e3c238be8
diff --git a/skyvern-frontend/src/routes/workflows/editor/debugger/getBlockParameterDependencies.test.ts b/skyvern-frontend/src/routes/workflows/editor/debugger/getBlockParameterDependencies.test.ts
new file mode 100644
index 000000000..0290c5f5c
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/editor/debugger/getBlockParameterDependencies.test.ts
@@ -0,0 +1,224 @@
+import { describe, expect, it } from "vitest";
+
+import { getBlockParameterDependencies } from "./getBlockParameterDependencies";
+import {
+ WorkflowParameterValueType,
+ type WorkflowBlock,
+ type WorkflowParameter,
+} from "@/routes/workflows/types/workflowTypes";
+
+function workflowParam(
+ key: string,
+ workflowParameterType: WorkflowParameter["workflow_parameter_type"] = "string",
+): WorkflowParameter {
+ return {
+ parameter_type: "workflow",
+ key,
+ description: null,
+ workflow_id: "wf-test",
+ workflow_parameter_id: `wpid-${key}`,
+ workflow_parameter_type: workflowParameterType,
+ default_value: null,
+ created_at: "",
+ modified_at: "",
+ deleted_at: null,
+ };
+}
+
+describe("getBlockParameterDependencies", () => {
+ it("returns all parameters when block is undefined", () => {
+ const params = [workflowParam("a"), workflowParam("b")];
+ expect(getBlockParameterDependencies(undefined, params)).toEqual(params);
+ });
+
+ it("returns all parameters when workflow has no parameters", () => {
+ const block = { block_type: "navigation", label: "n" } as WorkflowBlock;
+ expect(getBlockParameterDependencies(block, [])).toEqual([]);
+ });
+
+ it("collects keys from parameter_keys", () => {
+ const params = [workflowParam("x"), workflowParam("y")];
+ const block = {
+ block_type: "navigation",
+ label: "n",
+ parameter_keys: ["y"],
+ url: null,
+ title: "",
+ navigation_goal: null,
+ error_code_mapping: null,
+ max_retries: 0,
+ max_steps_per_run: null,
+ complete_on_download: false,
+ download_suffix: null,
+ totp_verification_url: null,
+ totp_identifier: null,
+ disable_cache: false,
+ complete_criterion: null,
+ terminate_criterion: null,
+ engine: null,
+ include_action_history_in_verification: false,
+ } as unknown as WorkflowBlock;
+
+ expect(getBlockParameterDependencies(block, params)).toEqual([params[1]]);
+ });
+
+ it("collects keys from jinja in string fields", () => {
+ const params = [workflowParam("portal_url"), workflowParam("other")];
+ const block = {
+ block_type: "navigation",
+ label: "n",
+ parameter_keys: [],
+ url: "{{ portal_url }}/login",
+ title: "",
+ navigation_goal: null,
+ error_code_mapping: null,
+ max_retries: 0,
+ max_steps_per_run: null,
+ complete_on_download: false,
+ download_suffix: null,
+ totp_verification_url: null,
+ totp_identifier: null,
+ disable_cache: false,
+ complete_criterion: null,
+ terminate_criterion: null,
+ engine: null,
+ include_action_history_in_verification: false,
+ } as unknown as WorkflowBlock;
+
+ expect(getBlockParameterDependencies(block, params)).toEqual([params[0]]);
+ });
+
+ it("parses jinja with filters", () => {
+ const params = [workflowParam("x")];
+ const block = {
+ block_type: "goto_url",
+ label: "u",
+ url: "{{ x | default('') }}",
+ } as unknown as WorkflowBlock;
+
+ expect(getBlockParameterDependencies(block, params)).toEqual([params[0]]);
+ });
+
+ it("collects workflow params referenced inside jinja expressions", () => {
+ const params = [workflowParam("count"), workflowParam("limit")];
+ const block = {
+ block_type: "conditional",
+ label: "branch",
+ branch_conditions: [
+ {
+ id: "1",
+ criteria: {
+ criteria_type: "jinja2_template",
+ expression: "{{ count < limit }}",
+ },
+ next_block_label: null,
+ description: null,
+ is_default: false,
+ },
+ ],
+ } as unknown as WorkflowBlock;
+
+ const result = getBlockParameterDependencies(block, params);
+ expect(result.map((p) => p.key).sort()).toEqual(["count", "limit"]);
+ });
+
+ it("collects workflow params from subscript jinja", () => {
+ const params = [workflowParam("items"), workflowParam("other")];
+ const block = {
+ block_type: "goto_url",
+ label: "u",
+ url: "{{ items[0] }}",
+ } as unknown as WorkflowBlock;
+
+ expect(getBlockParameterDependencies(block, params)).toEqual([params[0]]);
+ });
+
+ it("resolves context parameters to their source workflow parameter", () => {
+ const params = [workflowParam("source_key"), workflowParam("other")];
+ const block = {
+ block_type: "navigation",
+ label: "n",
+ parameter_keys: [],
+ parameters: [
+ {
+ parameter_type: "context",
+ key: "ctx_alias",
+ description: null,
+ source: workflowParam("source_key"),
+ value: null,
+ },
+ ],
+ url: null,
+ title: "",
+ navigation_goal: null,
+ error_code_mapping: null,
+ max_retries: 0,
+ max_steps_per_run: null,
+ complete_on_download: false,
+ download_suffix: null,
+ totp_verification_url: null,
+ totp_identifier: null,
+ disable_cache: false,
+ complete_criterion: null,
+ terminate_criterion: null,
+ engine: null,
+ include_action_history_in_verification: false,
+ } as unknown as WorkflowBlock;
+
+ expect(getBlockParameterDependencies(block, params)).toEqual([params[0]]);
+ });
+
+ it("returns empty when nothing on the block matches workflow parameters", () => {
+ const params = [workflowParam("only_in_workflow")];
+ const block = {
+ block_type: "goto_url",
+ label: "u",
+ url: "https://example.com",
+ } as unknown as WorkflowBlock;
+
+ expect(getBlockParameterDependencies(block, params)).toEqual([]);
+ });
+
+ it("only includes credential params attached to the login block", () => {
+ const params = [
+ workflowParam("u", WorkflowParameterValueType.String),
+ workflowParam("c1", WorkflowParameterValueType.CredentialId),
+ workflowParam("c2", WorkflowParameterValueType.CredentialId),
+ ];
+ const block = {
+ block_type: "login",
+ label: "login",
+ parameter_keys: ["u"],
+ parameters: [workflowParam("c1")],
+ url: null,
+ title: "",
+ navigation_goal: null,
+ error_code_mapping: null,
+ max_retries: 0,
+ max_steps_per_run: null,
+ totp_verification_url: null,
+ totp_identifier: null,
+ disable_cache: false,
+ complete_criterion: null,
+ terminate_criterion: null,
+ engine: null,
+ } as unknown as WorkflowBlock;
+
+ const result = getBlockParameterDependencies(block, params);
+ expect(result.map((p) => p.key).sort()).toEqual(["c1", "u"]);
+ });
+
+ it("includes loop_over key for for_loop blocks", () => {
+ const params = [workflowParam("items"), workflowParam("other")];
+ const block = {
+ block_type: "for_loop",
+ label: "loop",
+ loop_over: workflowParam("items"),
+ loop_blocks: [],
+ loop_variable_reference: null,
+ complete_if_empty: false,
+ } as unknown as WorkflowBlock;
+
+ expect(getBlockParameterDependencies(block, params)).toEqual([params[0]]);
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/editor/debugger/getBlockParameterDependencies.ts b/skyvern-frontend/src/routes/workflows/editor/debugger/getBlockParameterDependencies.ts
new file mode 100644
index 000000000..18b3ac8d7
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/editor/debugger/getBlockParameterDependencies.ts
@@ -0,0 +1,212 @@
+import type {
+ ContextParameter,
+ Parameter,
+ WorkflowBlock,
+ WorkflowParameter,
+} from "@/routes/workflows/types/workflowTypes";
+
+const JINJA_VARIABLE_REGEX = /\{\{\s*([^}|]+?)\s*(?:\|[^}]*)?\}\}/g;
+const SIMPLE_JINJA_PATH_REGEX =
+ /^[a-zA-Z_][a-zA-Z0-9_]*(?:\.[a-zA-Z_][a-zA-Z0-9_]*)*$/;
+const JINJA_IDENTIFIER_REGEX = /[a-zA-Z_][a-zA-Z0-9_]*/g;
+
+const MAX_RECURSION_DEPTH = 30;
+
+function extractJinjaRootsFromString(
+ text: string,
+ workflowParamKeys: Set<string>,
+): string[] {
+ const roots = new Set<string>();
+ JINJA_VARIABLE_REGEX.lastIndex = 0;
+ let match: RegExpExecArray | null;
+ while ((match = JINJA_VARIABLE_REGEX.exec(text)) !== null) {
+ const inner = match[1]?.trim();
+ if (!inner) {
+ continue;
+ }
+
+ if (SIMPLE_JINJA_PATH_REGEX.test(inner)) {
+ const root = inner.split(".")[0]?.trim();
+ if (root) {
+ roots.add(root);
+ }
+ continue;
+ }
+
+ JINJA_IDENTIFIER_REGEX.lastIndex = 0;
+ let identifierMatch: RegExpExecArray | null;
+ while ((identifierMatch = JINJA_IDENTIFIER_REGEX.exec(inner)) !== null) {
+ const identifier = identifierMatch[0];
+ if (workflowParamKeys.has(identifier)) {
+ roots.add(identifier);
+ }
+ }
+ }
+ return [...roots];
+}
+
+function collectTemplateStringsFromValue(
+ value: unknown,
+ depth: number,
+ skipLoopBodies: boolean,
+): string[] {
+ if (depth > MAX_RECURSION_DEPTH || value === null || value === undefined) {
+ return [];
+ }
+
+ if (typeof value === "string") {
+ return [value];
+ }
+
+ if (Array.isArray(value)) {
+ return value.flatMap((item) =>
+ collectTemplateStringsFromValue(item, depth + 1, skipLoopBodies),
+ );
+ }
+
+ if (typeof value !== "object") {
+ return [];
+ }
+
+ const record = value as Record<string, unknown>;
+ const isLoopContainer =
+ record.block_type === "for_loop" || record.block_type === "while_loop";
+
+ const strings: string[] = [];
+ for (const [key, child] of Object.entries(record)) {
+ if (key === "output_parameter") {
+ continue;
+ }
+ if (key === "parameters") {
+ continue;
+ }
+ if (skipLoopBodies && isLoopContainer && key === "loop_blocks") {
+ continue;
+ }
+ strings.push(
+ ...collectTemplateStringsFromValue(child, depth + 1, skipLoopBodies),
+ );
+ }
+ return strings;
+}
+
+function collectReferencedKeysFromJinja(
+ block: WorkflowBlock,
+ workflowParamKeys: Set<string>,
+): Set<string> {
+ const keys = new Set<string>();
+ for (const text of collectTemplateStringsFromValue(block, 0, true)) {
+ for (const root of extractJinjaRootsFromString(text, workflowParamKeys)) {
+ keys.add(root);
+ }
+ }
+ return keys;
+}
+
+function readExplicitParameterKeys(block: WorkflowBlock): string[] {
+ const raw = (block as { parameter_keys?: unknown }).parameter_keys;
+ if (!Array.isArray(raw)) {
+ return [];
+ }
+ return raw.filter((k): k is string => typeof k === "string" && k.length > 0);
+}
+
+function resolveWorkflowParameterKey(param: Parameter): string | null {
+ if (param.parameter_type === "workflow") {
+ return param.key;
+ }
+ if (param.parameter_type === "context") {
+ return resolveWorkflowParameterKey(
+ (param as ContextParameter).source as Parameter,
+ );
+ }
+ return null;
+}
+
+function readParameterKeysFromBlockParameters(block: WorkflowBlock): string[] {
+ if (!("parameters" in block) || !Array.isArray(block.parameters)) {
+ return [];
+ }
+ const keys: string[] = [];
+ for (const p of block.parameters) {
+ const workflowKey = resolveWorkflowParameterKey(p);
+ if (workflowKey) {
+ keys.push(workflowKey);
+ }
+ }
+ return keys;
+}
+
+function collectLoopAndBranchKeys(
+ block: WorkflowBlock,
+ workflowParamKeys: Set<string>,
+): string[] {
+ const keys: string[] = [];
+
+ if (block.block_type === "for_loop" && block.loop_over?.key) {
+ const loopKey = resolveWorkflowParameterKey(block.loop_over);
+ if (loopKey) {
+ keys.push(loopKey);
+ }
+ }
+
+ if (block.block_type === "while_loop" && block.condition?.expression) {
+ keys.push(
+ ...extractJinjaRootsFromString(
+ block.condition.expression,
+ workflowParamKeys,
+ ),
+ );
+ }
+
+ if (block.block_type === "conditional") {
+ for (const branch of block.branch_conditions) {
+ if (branch.criteria?.expression) {
+ keys.push(
+ ...extractJinjaRootsFromString(
+ branch.criteria.expression,
+ workflowParamKeys,
+ ),
+ );
+ }
+ }
+ }
+
+ return keys;
+}
+
+function collectDependencyKeySet(
+ block: WorkflowBlock,
+ workflowParamKeys: Set<string>,
+): Set<string> {
+ const keys = new Set<string>();
+
+ for (const k of readExplicitParameterKeys(block)) {
+ keys.add(k);
+ }
+ for (const k of readParameterKeysFromBlockParameters(block)) {
+ keys.add(k);
+ }
+ for (const k of collectReferencedKeysFromJinja(block, workflowParamKeys)) {
+ keys.add(k);
+ }
+ for (const k of collectLoopAndBranchKeys(block, workflowParamKeys)) {
+ keys.add(k);
+ }
+
+ return keys;
+}
+
+export function getBlockParameterDependencies(
+ block: WorkflowBlock | undefined,
+ workflowParameters: WorkflowParameter[],
+): WorkflowParameter[] {
+ if (!block || workflowParameters.length === 0) {
+ return workflowParameters;
+ }
+
+ const workflowParamKeys = new Set(workflowParameters.map((p) => p.key));
+ const wanted = collectDependencyKeySet(block, workflowParamKeys);
+
+ return workflowParameters.filter((p) => wanted.has(p.key));
+}
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx
index ff5ea3b93..9ca690075 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx
@@ -31,6 +31,8 @@ import {
type WorkflowParameter,
type Parameter,
} from "@/routes/workflows/types/workflowTypes";
+import { getBlockParameterDependencies } from "@/routes/workflows/editor/debugger/getBlockParameterDependencies";
+import { findWorkflowBlockByLabel } from "@/routes/workflows/workflowBlockUtils";
import { getInitialValues } from "@/routes/workflows/utils";
import { useDebuggerLastRunValuesStore } from "@/store/DebuggerLastRunValuesStore";
import { useBlockOutputStore } from "@/store/BlockOutputStore";
@@ -395,7 +397,17 @@ function NodeHeader({
(parameter) => parameter.parameter_type === "workflow",
);
- const parameters = getInitialValues(location, workflowParameters ?? []);
+ const lastRunValues = workflowPermanentId
+ ? useDebuggerLastRunValuesStore
+ .getState()
+ .getLastRunValues(workflowPermanentId)
+ : null;
+
+ const parameters = getInitialValues(
+ location,
+ workflowParameters ?? [],
+ lastRunValues,
+ );
// Merge with parameter overrides if provided
const mergedParameters = opts?.parameterOverrides
@@ -563,8 +575,6 @@ function NodeHeader({
workflow?.workflow_definition?.parameters ?? []
).filter(isWorkflowParameter);
- // If there are any workflow parameters, always prompt the user
- // The backend requires all params to be specified for each run
if (workflowParameters.length > 0) {
const lastRunValues = workflowPermanentId
? useDebuggerLastRunValuesStore
@@ -576,13 +586,26 @@ function NodeHeader({
workflowParameters,
lastRunValues,
);
- setCurrentParamValues(currentValues);
- setParametersToPrompt(workflowParameters);
- setShowParamsDialog(true);
+ const block = findWorkflowBlockByLabel(blocks, blockLabel);
+ const parametersToRun = getBlockParameterDependencies(
+ block ?? undefined,
+ workflowParameters,
+ );
+
+ if (parametersToRun.length > 0) {
+ setCurrentParamValues(currentValues);
+ setParametersToPrompt(parametersToRun);
+ setShowParamsDialog(true);
+ return;
+ }
+
+ runBlock.mutate({
+ codeGen: numBlocksInWorkflow === 1,
+ parameterOverrides: currentValues,
+ });
return;
}
- // No parameters, run directly
runBlock.mutate({ codeGen: numBlocksInWorkflow === 1 });
};
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index f8090e12f..44a33fc33 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -673,6 +673,14 @@ class AgentFunction:
"""
return None
+ async def wait_for_challenge_solver(self, page: Page) -> None:
+ """Wait for a cloud-managed challenge solver if one is attached to the page.
+
+ OSS no-op. Cloud overrides this so OSS-synced browser/action code can
+ wait at navigation/action boundaries without importing cloud modules.
+ """
+ return None
+
async def should_shadow_extraction_cache_hit(self, task: Task) -> bool:
"""Cloud-overridable sample gate for extract-information shadow mode. OSS no-op."""
return False
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 70a71375f..fb2e80f9c 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -864,6 +864,7 @@ class ActionHandler:
page: Page,
action: Action,
) -> list[ActionResult]:
+ await app.AGENT_FUNCTION.wait_for_challenge_solver(page=page)
LOG.info(
"Handling action",
action_type=action.action_type,
@@ -894,6 +895,7 @@ class ActionHandler:
handler = ActionHandler._handled_action_types[action.action_type]
results = await handler(action, page, scraped_page, task, step)
actions_result.extend(results)
+ await app.AGENT_FUNCTION.wait_for_challenge_solver(page=page)
# do the teardown
teardown = ActionHandler._teardown_action_types.get(action.action_type)
if teardown:
diff --git a/skyvern/webeye/real_browser_state.py b/skyvern/webeye/real_browser_state.py
index bd6462f31..8ea10ba29 100644
--- a/skyvern/webeye/real_browser_state.py
+++ b/skyvern/webeye/real_browser_state.py
@@ -18,6 +18,7 @@ from skyvern.exceptions import (
FailedToStopLoadingPage,
MissingBrowserStatePage,
)
+from skyvern.forge import app
from skyvern.forge.sdk.trace import traced
from skyvern.schemas.runs import ProxyLocationInput
from skyvern.webeye.browser_artifacts import BrowserArtifacts, VideoArtifact
@@ -149,6 +150,10 @@ class RealBrowserState(BrowserState):
settle=self._wait_for_settle,
wait_until=wait_until,
)
+ await self._wait_for_challenge_solver(page=page)
+
+ async def _wait_for_challenge_solver(self, page: Page) -> None:
+ await app.AGENT_FUNCTION.wait_for_challenge_solver(page=page)
async def get_working_page(self) -> Page | None:
# HACK: currently, assuming the last page is always the working page.
@@ -381,6 +386,7 @@ class RealBrowserState(BrowserState):
loading_time=end_time - start_time,
)
await self._wait_for_settle()
+ await self._wait_for_challenge_solver(page=page)
except Exception as e:
LOG.exception(f"Error while reload url: {repr(e)}")
raise FailedToReloadPage(url=page.url, error_message=repr(e))
@@ -400,6 +406,10 @@ class RealBrowserState(BrowserState):
wait_seconds: float = 0,
must_included_tags: list[str] | None = None,
) -> ScrapedPage:
+ page = await self.get_working_page()
+ if page is not None:
+ await self._wait_for_challenge_solver(page=page)
+
return await scraper.scrape_website(
browser_state=self,
url=url,