Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Get it back - Data schema spec fix: TSON instead of JSON (#3566) (#3569)
Shuchang Zheng94c05e092abf
diff --git a/skyvern-frontend/src/components/DataSchemaInputGroup/WorkflowDataSchemaInputGroup.tsx b/skyvern-frontend/src/components/DataSchemaInputGroup/WorkflowDataSchemaInputGroup.tsxindex a2ac5ba4d..2d5d17577 100644--- a/skyvern-frontend/src/components/DataSchemaInputGroup/WorkflowDataSchemaInputGroup.tsx+++ b/skyvern-frontend/src/components/DataSchemaInputGroup/WorkflowDataSchemaInputGroup.tsx@@ -17,6 +17,7 @@ import { AutoResizingTextarea } from "../AutoResizingTextarea/AutoResizingTextarimport { Button } from "../ui/button";import { AxiosError } from "axios";import { toast } from "../ui/use-toast";+import { TSON } from "@/util/tson";import { cn } from "@/util/utils";type Props = {@@ -36,42 +37,13 @@ function WorkflowDataSchemaInputGroup({const [generateWithAIActive, setGenerateWithAIActive] = useState(false);const [generateWithAIPrompt, setGenerateWithAIPrompt] = useState("");- function computeJsonError(- jsonText: string,- ): { message: string; line?: number; column?: number } | null {- try {- JSON.parse(jsonText);- return null;- } catch (e) {- const message = e instanceof Error ? e.message : "Invalid JSON";- // Try to extract position and compute line/column for friendlier feedback- const match = message.match(/position\s+(\d+)/i);- if (!match) {- return { message };- }- const pos = Number(match[1]);- if (Number.isNaN(pos)) {- return { message };- }- let line = 1;- let col = 1;- for (let i = 0; i < Math.min(pos, jsonText.length); i++) {- if (jsonText[i] === "\n") {- line += 1;- col = 1;- } else {- col += 1;- }- }- return { message, line, column: col };- }- }-- const jsonError = useMemo(() => {+ const tsonResult = useMemo(() => {if (value === "null") return null;- return computeJsonError(value);+ return TSON.parse(value);}, [value]);+ console.log({ tsonResult });+const getDataSchemaSuggestionMutation = useMutation({mutationFn: async () => {const client = await getClient(credentialGetter);@@ -161,7 +133,9 @@ function WorkflowDataSchemaInputGroup({<divclassName={cn("rounded-md",- jsonError ? "ring-1 ring-red-500" : undefined,+ tsonResult && !tsonResult.success+ ? "ring-1 ring-red-500"+ : undefined,)}><CodeEditor@@ -172,12 +146,8 @@ function WorkflowDataSchemaInputGroup({fontSize={8}/></div>- {jsonError && (- <div className="text-xs text-red-400">- {jsonError.line && jsonError.column- ? `Invalid JSON (${jsonError.line}:${jsonError.column}) — ${jsonError.message}`- : `Invalid JSON — ${jsonError.message}`}- </div>+ {tsonResult !== null && !tsonResult.success && tsonResult.error && (+ <div className="text-xs text-red-400">{tsonResult.error}</div>)}</div>)}diff --git a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.tsindex d1bf316ae..38ae1a0da 100644--- a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts+++ b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts@@ -2,6 +2,9 @@ import Dagre from "@dagrejs/dagre";import type { Node } from "@xyflow/react";import { Edge } from "@xyflow/react";import { nanoid } from "nanoid";++import { TSON } from "@/util/tson";+import {WorkflowBlockType,WorkflowBlockTypes,@@ -2188,10 +2191,12 @@ function getWorkflowErrors(nodes: Array<AppNode>): Array<string> {}// Validate Task data schema JSON when enabled (value different from "null")if (node.data.dataSchema && node.data.dataSchema !== "null") {- try {- JSON.parse(node.data.dataSchema);- } catch {- errors.push(`${node.data.label}: Data schema is not valid JSON.`);+ const result = TSON.parse(node.data.dataSchema);++ if (!result.success) {+ errors.push(+ `${node.data.label}: Data schema has invalid templated JSON: ${result.error ?? "-"}`,+ );}}});@@ -2227,10 +2232,12 @@ function getWorkflowErrors(nodes: Array<AppNode>): Array<string> {}// Validate Extraction data schema JSON when enabled (value different from "null")if (node.data.dataSchema && node.data.dataSchema !== "null") {- try {- JSON.parse(node.data.dataSchema);- } catch {- errors.push(`${node.data.label}: Data schema is not valid JSON.`);+ const result = TSON.parse(node.data.dataSchema);++ if (!result.success) {+ errors.push(+ `${node.data.label}: Data schema has invalid templated JSON: ${result.error ?? "-"}`,+ );}}});diff --git a/skyvern-frontend/src/util/tson.test.ts b/skyvern-frontend/src/util/tson.test.tsnew file mode 100644index 000000000..adc92949d--- /dev/null+++ b/skyvern-frontend/src/util/tson.test.ts@@ -0,0 +1,91 @@+// TSON.parse.test.ts+import { TSON } from "./tson";+import { describe, test, expect } from "vitest";++describe("TSON.parse", () => {+ test("single top-level template works", () => {+ const input = "{{ hello }}";+ const result = TSON.parse(input);++ expect(result.success).toBe(true);+ expect(result.data).toEqual("<STUB>");+ });++ test("preserves double braces inside quoted strings", () => {+ const input = '{"a": "{{ hello }}"}';+ const result = TSON.parse(input);++ expect(result.success).toBe(true);+ expect(result.data).toEqual({ a: "{{ hello }}" });+ });++ test("replaces double braces outside strings with stub", () => {+ const input = '{"a": {{ hello }} }';+ const result = TSON.parse(input);++ expect(result.success).toBe(true);+ expect(result.data).toEqual({ a: "<STUB>" });+ });++ test("handles double braces in keys and values", () => {+ const input = `+{+ "hello": "world",+ {{foo}}: "bar",+ "baz": {{quux}}+}`;+ const result = TSON.parse(input);++ expect(result.success).toBe(true);+ expect(result.data).toEqual({+ hello: "world",+ "<STUB>": "bar",+ baz: "<STUB>",+ });+ });++ test("does not allow trailing commas", () => {+ const input = `+{+ "hello": "world",+ {{foo}}: "bar",+ "baz": {{quux}},+}`;+ const result = TSON.parse(input);++ expect(result.success).toBe(false);+ expect(result.error).toContain("Expected double-quoted property name");+ });++ test("detects unclosed double braces", () => {+ const input = "{{ unclosed";+ const result = TSON.parse(input);++ expect(result.success).toBe(false);+ expect(result.error).toContain("Unclosed");+ });++ test("detects unmatched closing double braces", () => {+ const input = "closed }}";+ const result = TSON.parse(input);++ expect(result.success).toBe(false);+ expect(result.error).toContain("Unmatched");+ });++ test("handles nested double braces", () => {+ const input = "{{ {{ nested }} }}";+ const result = TSON.parse(input);++ expect(result.success).toBe(true);+ expect(result.data).toEqual("<STUB>");+ });++ test("handles double braces in arrays", () => {+ const input = '[{{ }}, {{ }}, "normal"]';+ const result = TSON.parse(input);++ expect(result.success).toBe(true);+ expect(result.data).toEqual(["<STUB>", "<STUB>", "normal"]);+ });+});diff --git a/skyvern-frontend/src/util/tson.ts b/skyvern-frontend/src/util/tson.tsnew file mode 100644index 000000000..dc01254a6--- /dev/null+++ b/skyvern-frontend/src/util/tson.ts@@ -0,0 +1,223 @@+type JSONValue =+ | string+ | number+ | boolean+ | null+ | JSONValue[]+ | { [key: string]: JSONValue };++interface ParseResult {+ success: boolean;+ data?: JSONValue;+ error?: string;+}++const placeholder = () => "<STUB>";++/**+ * TSON ("templated JSON") is a superset of JSON, where double curly braces {{...}} can:+ * - exist anywhere outside of string literals, and+ * - are treated as placeholders+ */+const TSON = {+ parse(input: string): ParseResult {+ try {+ const balanceCheck = checkDoubleBraceBalance(input);++ if (!balanceCheck.balanced) {+ return {+ success: false,+ error: balanceCheck.error,+ };+ }++ const pipeline = [+ replaceBracesOutsideQuotes,+ // removeTrailingCommas,+ JSON.parse,+ ];++ const parsed = pipeline.reduce((acc, fn) => fn(acc), input) as JSONValue;++ return {+ success: true,+ data: parsed,+ };+ } catch (error) {+ return {+ success: false,+ error: error instanceof Error ? error.message : String(error),+ };+ }+ },+};++function checkDoubleBraceBalance(input: string): {+ balanced: boolean;+ error?: string;+} {+ let inString = false;+ let escapeNext = false;+ let depth = 0;++ for (let i = 0; i < input.length; i++) {+ const char = input[i];+ const nextChar = input[i + 1];++ // handle escape sequences+ if (escapeNext) {+ escapeNext = false;+ continue;+ }++ if (char === "\\") {+ escapeNext = true;+ continue;+ }++ // inside-string tracking+ if (char === '"') {+ inString = !inString;+ continue;+ }++ // double braces counts (only outside strings)+ if (!inString) {+ if (char === "{" && nextChar === "{") {+ depth++;+ i++; // skip next char+ } else if (char === "}" && nextChar === "}") {+ depth--;+ if (depth < 0) {+ return {+ balanced: false,+ error: `Unmatched closing }} at position ${i}`,+ };+ }+ i++; // skip next char+ }+ }+ }++ if (depth > 0) {+ return {+ balanced: false,+ error: `Unclosed {{ - missing ${depth} closing }}`,+ };+ }++ return { balanced: true };+}++function replaceBracesOutsideQuotes(input: string): string {+ let result = "";+ let inString = false;+ let escapeNext = false;+ let inDoubleBrace = 0; // track nesting depth of {{...}}++ for (let i = 0; i < input.length; i++) {+ const char = input[i];+ const nextChar = input[i + 1];++ // escape sequences+ if (escapeNext) {+ if (inDoubleBrace === 0) {+ result += char;+ }+ escapeNext = false;+ continue;+ }++ if (char === "\\") {+ if (inDoubleBrace === 0) {+ result += char;+ }+ escapeNext = true;+ continue;+ }++ // inside-string tracking+ if (char === '"') {+ inString = !inString;+ if (inDoubleBrace === 0) {+ result += char;+ }+ continue;+ }++ // double braces (only outside strings)+ if (!inString) {+ if (char === "{" && nextChar === "{") {+ if (inDoubleBrace === 0) {+ result += `"${placeholder()}"`;+ }+ inDoubleBrace++;+ i++; // skip next char+ continue;+ } else if (char === "}" && nextChar === "}") {+ inDoubleBrace--;+ i++; // skip next char+ continue;+ }+ }++ // add characters when we're not inside double braces+ if (inDoubleBrace === 0) {+ result += char;+ }+ }++ return result;+}++export function removeTrailingCommas(input: string): string {+ let result = "";+ let inString = false;+ let escapeNext = false;++ for (let i = 0; i < input.length; i++) {+ const char = input[i];++ // escape sequences+ if (escapeNext) {+ result += char;+ escapeNext = false;+ continue;+ }++ if (char === "\\") {+ result += char;+ escapeNext = true;+ continue;+ }++ // inside-string tracking+ if (char === '"') {+ inString = !inString;+ result += char;+ continue;+ }++ // check for trailing commas (outside strings)+ if (!inString && char === ",") {+ // look-ahead for the next non-whitespace character+ let j = i + 1;+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment+ // @ts-ignore+ while (j < input.length && /\s/.test(input[j])) {+ j++;+ }++ // if next non-whitespace is } or ], skip the comma+ if (j < input.length && (input[j] === "}" || input[j] === "]")) {+ continue; // Skip this comma+ }+ }++ result += char;+ }++ return result;+}++export { TSON };