Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
integrate parameters to support legacy tasks with parameters (#3567)
Shuchang Zhengbb1e7316a355
diff --git a/skyvern-frontend/src/components/DataSchemaInputGroup/WorkflowDataSchemaInputGroup.tsx b/skyvern-frontend/src/components/DataSchemaInputGroup/WorkflowDataSchemaInputGroup.tsxindex 2d5d17577..a2ac5ba4d 100644--- a/skyvern-frontend/src/components/DataSchemaInputGroup/WorkflowDataSchemaInputGroup.tsx+++ b/skyvern-frontend/src/components/DataSchemaInputGroup/WorkflowDataSchemaInputGroup.tsx@@ -17,7 +17,6 @@ import { AutoResizingTextarea } from "../AutoResizingTextarea/AutoResizingTextarimport { Button } from "../ui/button";import { AxiosError } from "axios";import { toast } from "../ui/use-toast";-import { TSON } from "@/util/tson";import { cn } from "@/util/utils";type Props = {@@ -37,13 +36,42 @@ function WorkflowDataSchemaInputGroup({const [generateWithAIActive, setGenerateWithAIActive] = useState(false);const [generateWithAIPrompt, setGenerateWithAIPrompt] = useState("");- const tsonResult = useMemo(() => {+ function computeJsonError(+ jsonText: string,+ ): { message: string; line?: number; column?: number } | null {+ try {+ JSON.parse(jsonText);+ return null;+ } catch (e) {+ const message = e instanceof Error ? e.message : "Invalid JSON";+ // Try to extract position and compute line/column for friendlier feedback+ const match = message.match(/position\s+(\d+)/i);+ if (!match) {+ return { message };+ }+ const pos = Number(match[1]);+ if (Number.isNaN(pos)) {+ return { message };+ }+ let line = 1;+ let col = 1;+ for (let i = 0; i < Math.min(pos, jsonText.length); i++) {+ if (jsonText[i] === "\n") {+ line += 1;+ col = 1;+ } else {+ col += 1;+ }+ }+ return { message, line, column: col };+ }+ }++ const jsonError = useMemo(() => {if (value === "null") return null;- return TSON.parse(value);+ return computeJsonError(value);}, [value]);- console.log({ tsonResult });-const getDataSchemaSuggestionMutation = useMutation({mutationFn: async () => {const client = await getClient(credentialGetter);@@ -133,9 +161,7 @@ function WorkflowDataSchemaInputGroup({<divclassName={cn("rounded-md",- tsonResult && !tsonResult.success- ? "ring-1 ring-red-500"- : undefined,+ jsonError ? "ring-1 ring-red-500" : undefined,)}><CodeEditor@@ -146,8 +172,12 @@ function WorkflowDataSchemaInputGroup({fontSize={8}/></div>- {tsonResult !== null && !tsonResult.success && tsonResult.error && (- <div className="text-xs text-red-400">{tsonResult.error}</div>+ {jsonError && (+ <div className="text-xs text-red-400">+ {jsonError.line && jsonError.column+ ? `Invalid JSON (${jsonError.line}:${jsonError.column}) — ${jsonError.message}`+ : `Invalid JSON — ${jsonError.message}`}+ </div>)}</div>)}diff --git a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.tsindex 38ae1a0da..d1bf316ae 100644--- a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts+++ b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts@@ -2,9 +2,6 @@ import Dagre from "@dagrejs/dagre";import type { Node } from "@xyflow/react";import { Edge } from "@xyflow/react";import { nanoid } from "nanoid";--import { TSON } from "@/util/tson";-import {WorkflowBlockType,WorkflowBlockTypes,@@ -2191,12 +2188,10 @@ function getWorkflowErrors(nodes: Array<AppNode>): Array<string> {}// Validate Task data schema JSON when enabled (value different from "null")if (node.data.dataSchema && node.data.dataSchema !== "null") {- const result = TSON.parse(node.data.dataSchema);-- if (!result.success) {- errors.push(- `${node.data.label}: Data schema has invalid templated JSON: ${result.error ?? "-"}`,- );+ try {+ JSON.parse(node.data.dataSchema);+ } catch {+ errors.push(`${node.data.label}: Data schema is not valid JSON.`);}}});@@ -2232,12 +2227,10 @@ function getWorkflowErrors(nodes: Array<AppNode>): Array<string> {}// Validate Extraction data schema JSON when enabled (value different from "null")if (node.data.dataSchema && node.data.dataSchema !== "null") {- const result = TSON.parse(node.data.dataSchema);-- if (!result.success) {- errors.push(- `${node.data.label}: Data schema has invalid templated JSON: ${result.error ?? "-"}`,- );+ try {+ JSON.parse(node.data.dataSchema);+ } catch {+ errors.push(`${node.data.label}: Data schema is not valid JSON.`);}}});diff --git a/skyvern-frontend/src/util/tson.test.ts b/skyvern-frontend/src/util/tson.test.tsdeleted file mode 100644index adc92949d..000000000--- a/skyvern-frontend/src/util/tson.test.ts+++ /dev/null@@ -1,91 +0,0 @@-// TSON.parse.test.ts-import { TSON } from "./tson";-import { describe, test, expect } from "vitest";--describe("TSON.parse", () => {- test("single top-level template works", () => {- const input = "{{ hello }}";- const result = TSON.parse(input);-- expect(result.success).toBe(true);- expect(result.data).toEqual("<STUB>");- });-- test("preserves double braces inside quoted strings", () => {- const input = '{"a": "{{ hello }}"}';- const result = TSON.parse(input);-- expect(result.success).toBe(true);- expect(result.data).toEqual({ a: "{{ hello }}" });- });-- test("replaces double braces outside strings with stub", () => {- const input = '{"a": {{ hello }} }';- const result = TSON.parse(input);-- expect(result.success).toBe(true);- expect(result.data).toEqual({ a: "<STUB>" });- });-- test("handles double braces in keys and values", () => {- const input = `-{- "hello": "world",- {{foo}}: "bar",- "baz": {{quux}}-}`;- const result = TSON.parse(input);-- expect(result.success).toBe(true);- expect(result.data).toEqual({- hello: "world",- "<STUB>": "bar",- baz: "<STUB>",- });- });-- test("does not allow trailing commas", () => {- const input = `-{- "hello": "world",- {{foo}}: "bar",- "baz": {{quux}},-}`;- const result = TSON.parse(input);-- expect(result.success).toBe(false);- expect(result.error).toContain("Expected double-quoted property name");- });-- test("detects unclosed double braces", () => {- const input = "{{ unclosed";- const result = TSON.parse(input);-- expect(result.success).toBe(false);- expect(result.error).toContain("Unclosed");- });-- test("detects unmatched closing double braces", () => {- const input = "closed }}";- const result = TSON.parse(input);-- expect(result.success).toBe(false);- expect(result.error).toContain("Unmatched");- });-- test("handles nested double braces", () => {- const input = "{{ {{ nested }} }}";- const result = TSON.parse(input);-- expect(result.success).toBe(true);- expect(result.data).toEqual("<STUB>");- });-- test("handles double braces in arrays", () => {- const input = '[{{ }}, {{ }}, "normal"]';- const result = TSON.parse(input);-- expect(result.success).toBe(true);- expect(result.data).toEqual(["<STUB>", "<STUB>", "normal"]);- });-});diff --git a/skyvern-frontend/src/util/tson.ts b/skyvern-frontend/src/util/tson.tsdeleted file mode 100644index dc01254a6..000000000--- a/skyvern-frontend/src/util/tson.ts+++ /dev/null@@ -1,223 +0,0 @@-type JSONValue =- | string- | number- | boolean- | null- | JSONValue[]- | { [key: string]: JSONValue };--interface ParseResult {- success: boolean;- data?: JSONValue;- error?: string;-}--const placeholder = () => "<STUB>";--/**- * TSON ("templated JSON") is a superset of JSON, where double curly braces {{...}} can:- * - exist anywhere outside of string literals, and- * - are treated as placeholders- */-const TSON = {- parse(input: string): ParseResult {- try {- const balanceCheck = checkDoubleBraceBalance(input);-- if (!balanceCheck.balanced) {- return {- success: false,- error: balanceCheck.error,- };- }-- const pipeline = [- replaceBracesOutsideQuotes,- // removeTrailingCommas,- JSON.parse,- ];-- const parsed = pipeline.reduce((acc, fn) => fn(acc), input) as JSONValue;-- return {- success: true,- data: parsed,- };- } catch (error) {- return {- success: false,- error: error instanceof Error ? error.message : String(error),- };- }- },-};--function checkDoubleBraceBalance(input: string): {- balanced: boolean;- error?: string;-} {- let inString = false;- let escapeNext = false;- let depth = 0;-- for (let i = 0; i < input.length; i++) {- const char = input[i];- const nextChar = input[i + 1];-- // handle escape sequences- if (escapeNext) {- escapeNext = false;- continue;- }-- if (char === "\\") {- escapeNext = true;- continue;- }-- // inside-string tracking- if (char === '"') {- inString = !inString;- continue;- }-- // double braces counts (only outside strings)- if (!inString) {- if (char === "{" && nextChar === "{") {- depth++;- i++; // skip next char- } else if (char === "}" && nextChar === "}") {- depth--;- if (depth < 0) {- return {- balanced: false,- error: `Unmatched closing }} at position ${i}`,- };- }- i++; // skip next char- }- }- }-- if (depth > 0) {- return {- balanced: false,- error: `Unclosed {{ - missing ${depth} closing }}`,- };- }-- return { balanced: true };-}--function replaceBracesOutsideQuotes(input: string): string {- let result = "";- let inString = false;- let escapeNext = false;- let inDoubleBrace = 0; // track nesting depth of {{...}}-- for (let i = 0; i < input.length; i++) {- const char = input[i];- const nextChar = input[i + 1];-- // escape sequences- if (escapeNext) {- if (inDoubleBrace === 0) {- result += char;- }- escapeNext = false;- continue;- }-- if (char === "\\") {- if (inDoubleBrace === 0) {- result += char;- }- escapeNext = true;- continue;- }-- // inside-string tracking- if (char === '"') {- inString = !inString;- if (inDoubleBrace === 0) {- result += char;- }- continue;- }-- // double braces (only outside strings)- if (!inString) {- if (char === "{" && nextChar === "{") {- if (inDoubleBrace === 0) {- result += `"${placeholder()}"`;- }- inDoubleBrace++;- i++; // skip next char- continue;- } else if (char === "}" && nextChar === "}") {- inDoubleBrace--;- i++; // skip next char- continue;- }- }-- // add characters when we're not inside double braces- if (inDoubleBrace === 0) {- result += char;- }- }-- return result;-}--export function removeTrailingCommas(input: string): string {- let result = "";- let inString = false;- let escapeNext = false;-- for (let i = 0; i < input.length; i++) {- const char = input[i];-- // escape sequences- if (escapeNext) {- result += char;- escapeNext = false;- continue;- }-- if (char === "\\") {- result += char;- escapeNext = true;- continue;- }-- // inside-string tracking- if (char === '"') {- inString = !inString;- result += char;- continue;- }-- // check for trailing commas (outside strings)- if (!inString && char === ",") {- // look-ahead for the next non-whitespace character- let j = i + 1;- // eslint-disable-next-line @typescript-eslint/ban-ts-comment- // @ts-ignore- while (j < input.length && /\s/.test(input[j])) {- j++;- }-- // if next non-whitespace is } or ], skip the comma- if (j < input.length && (input[j] === "}" || input[j] === "]")) {- continue; // Skip this comma- }- }-- result += char;- }-- return result;-}--export { TSON };diff --git a/skyvern-frontend/vitest.config.ts b/skyvern-frontend/vitest.config.tsnew file mode 100644index 000000000..274f0b866--- /dev/null+++ b/skyvern-frontend/vitest.config.ts@@ -0,0 +1,15 @@+import { defineConfig } from 'vitest/config'+import react from '@vitejs/plugin-react-swc'+import path from 'path'++export default defineConfig({+ plugins: [react()],+ test: {+ passWithNoTests: true,+ },+ resolve: {+ alias: {+ '@': path.resolve(__dirname, './src'),+ },+ },+})diff --git a/skyvern/core/script_generations/generate_script.py b/skyvern/core/script_generations/generate_script.pyindex c501e24b7..7c37752fa 100644--- a/skyvern/core/script_generations/generate_script.py+++ b/skyvern/core/script_generations/generate_script.py@@ -1350,6 +1350,18 @@ def __build_base_task_statement() -> list[cst.Arg]:block_type = block.get("block_type")prompt = block.get("prompt") if block_type == "task_v2" else block.get("navigation_goal")+ # add parameters to prompt+ parameters = block.get("parameters", [])+ navigation_payload = {}+ # make all parameters as jinja2 template parameters in the generated code+ for parameter in parameters:+ parameter_key = parameter["key"]+ navigation_payload[parameter_key] = "{{" + parameter_key + "}}"++ if navigation_payload:+ prompt = prompt or ""+ prompt = f"{prompt}\n{navigation_payload}"+args = [cst.Arg(keyword=cst.Name("prompt"),diff --git a/skyvern/core/script_generations/skyvern_page.py b/skyvern/core/script_generations/skyvern_page.pyindex a796698b8..9f3ebd95c 100644--- a/skyvern/core/script_generations/skyvern_page.py+++ b/skyvern/core/script_generations/skyvern_page.py@@ -509,7 +509,7 @@ class SkyvernPage:if ai_infer and intention:try:prompt = context.prompt if context else None- data = _get_context_data(data)+ data = data or {}if (totp_identifier or totp_url) and context and organization_id and task_id:verification_code = await poll_verification_code(organization_id=organization_id,@@ -535,7 +535,6 @@ class SkyvernPage:script_generation_input_text_prompt = prompt_engine.load_prompt(template="script-generation-input-text-generatiion",intention=intention,- data=data,goal=prompt,)json_response = await app.SINGLE_INPUT_AGENT_LLM_API_HANDLER(@@ -620,7 +619,8 @@ class SkyvernPage:if ai_infer and intention and task and step:try:prompt = context.prompt if context else None- data = _get_context_data(data)+ # data = _get_context_data(data)+ data = data or {}refreshed_page = await self.scraped_page.generate_scraped_page_without_screenshots()self.scraped_page = refreshed_pageelement_tree = refreshed_page.build_element_tree()diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.pyindex 1e0ee7afe..8634ba5dc 100644--- a/skyvern/services/script_service.py+++ b/skyvern/services/script_service.py@@ -1114,6 +1114,7 @@ async def run_task(prompt: str,url: str | None = None,max_steps: int | None = None,+ download_suffix: str | None = None,totp_identifier: str | None = None,totp_url: str | None = None,label: str | None = None,@@ -1133,6 +1134,7 @@ async def run_task(url=url,label=cache_key,)+ prompt = _render_template_with_label(prompt, cache_key)# set the prompt in the RunContextcontext = skyvern_context.ensure_context()context.prompt = prompt@@ -1190,6 +1192,7 @@ async def download(prompt: str,url: str | None = None,complete_on_download: bool = True,+ download_suffix: str | None = None,max_steps: int | None = None,totp_identifier: str | None = None,totp_url: str | None = None,@@ -1198,7 +1201,6 @@ async def download() -> None:cache_key = cache_key or labelcached_fn = script_run_context_manager.get_cached_fn(cache_key)-context: skyvern_context.SkyvernContext | Noneif cache_key and cached_fn:# Auto-create workflow block run and task if workflow_run_id is available@@ -1208,6 +1210,7 @@ async def download(url=url,label=cache_key,)+ prompt = _render_template_with_label(prompt, cache_key)# set the prompt in the RunContextcontext = skyvern_context.ensure_context()context.prompt = prompt@@ -1265,6 +1268,7 @@ async def action(prompt: str,url: str | None = None,max_steps: int | None = None,+ download_suffix: str | None = None,totp_identifier: str | None = None,totp_url: str | None = None,label: str | None = None,@@ -1281,6 +1285,7 @@ async def action(url=url,label=cache_key,)+ prompt = _render_template_with_label(prompt, cache_key)# set the prompt in the RunContextcontext = skyvern_context.ensure_context()context.prompt = prompt@@ -1346,12 +1351,14 @@ async def login(cached_fn = script_run_context_manager.get_cached_fn(cache_key)if cache_key and cached_fn:# Auto-create workflow block run and task if workflow_run_id is available+ # render template with labelworkflow_run_block_id, task_id, step_id = await _create_workflow_block_run_and_task(block_type=BlockType.LOGIN,prompt=prompt,url=url,label=cache_key,)+ prompt = _render_template_with_label(prompt, cache_key)# set the prompt in the RunContextcontext = skyvern_context.ensure_context()context.prompt = prompt@@ -1424,6 +1431,7 @@ async def extract(url=url,label=cache_key,)+ prompt = _render_template_with_label(prompt, cache_key)# set the prompt in the RunContextcontext = skyvern_context.ensure_context()context.prompt = prompt@@ -1546,27 +1554,27 @@ def _render_template_with_label(template: str, label: str | None = None) -> str:context = skyvern_context.current()if context and context.workflow_run_id:workflow_run_context = app.WORKFLOW_CONTEXT_MANAGER.get_workflow_run_context(context.workflow_run_id)- block_reference_data: dict[str, Any] = workflow_run_context.get_block_metadata(label)template_data = workflow_run_context.values.copy()- if label in template_data:- current_value = template_data[label]- if isinstance(current_value, dict):- block_reference_data.update(current_value)- else:- LOG.warning(- f"Script service: Parameter {label} has a registered reference value, going to overwrite it by block metadata"- )-if label:+ block_reference_data = workflow_run_context.get_block_metadata(label)+ if label in template_data:+ current_value = template_data[label]+ if isinstance(current_value, dict):+ block_reference_data.update(current_value)+ else:+ LOG.warning(+ f"Script service: Parameter {label} has a registered reference value, going to overwrite it by block metadata"+ )+template_data[label] = block_reference_data- # inject the forloop metadata as global variables- if "current_index" in block_reference_data:- template_data["current_index"] = block_reference_data["current_index"]- if "current_item" in block_reference_data:- template_data["current_item"] = block_reference_data["current_item"]- if "current_value" in block_reference_data:- template_data["current_value"] = block_reference_data["current_value"]+ # inject the forloop metadata as global variables+ if "current_index" in block_reference_data:+ template_data["current_index"] = block_reference_data["current_index"]+ if "current_item" in block_reference_data:+ template_data["current_item"] = block_reference_data["current_item"]+ if "current_value" in block_reference_data:+ template_data["current_value"] = block_reference_data["current_value"]return render_template(template, data=template_data)