Skip to content

Automate browser based workflows with AI

fix: enforce LLM block schema outputs (#6965)

Suchintan4ee1e133656d
diff --git a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
index 4dc059c4a..e9b24497b 100644
--- a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
@@ -103,6 +103,7 @@ import { DebuggerBlockRuns } from "@/routes/workflows/debugger/DebuggerBlockRuns
import { copyText } from "@/util/copyText";
import { isMacPlatform } from "@/util/platform";
import { parseHeaderJson } from "@/util/secretHeaders";
+import { getJsonParseErrorDetail } from "@/util/jsonParseError";
import { cn } from "@/util/utils";
import { FlowRenderer, type FlowRendererProps } from "./FlowRenderer";
@@ -2432,7 +2433,10 @@ function Workspace({
} catch (error) {
toast({
title: "Error",
- description: "Invalid JSON format in extra http headers",
+ description: `Invalid JSON format in extra http headers: ${getJsonParseErrorDetail(
+ saveData.settings.extraHttpHeaders ?? "",
+ error,
+ )}`,
variant: "destructive",
});
return;
@@ -2448,7 +2452,10 @@ function Workspace({
} catch (error) {
toast({
title: "Error",
- description: "Invalid JSON format in cdp connect headers",
+ description: `Invalid JSON format in cdp connect headers: ${getJsonParseErrorDetail(
+ saveData.settings.cdpConnectHeaders ?? "",
+ error,
+ )}`,
variant: "destructive",
});
return;
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/FileParserEditor.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/FileParserEditor.tsx
index 2eac9a9c0..8037f452f 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/FileParserEditor.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/FileParserEditor.tsx
@@ -2,6 +2,12 @@ import { useReactFlow } from "@xyflow/react";
import { HelpTooltip } from "@/components/HelpTooltip";
import { ModelSelector } from "@/components/ModelSelector";
+import {
+ Accordion,
+ AccordionContent,
+ AccordionItem,
+ AccordionTrigger,
+} from "@/components/ui/accordion";
import { Label } from "@/components/ui/label";
import {
Select,
@@ -153,13 +159,26 @@ function FileParserEditorBody({
value={data.model}
onChange={(value) => update({ model: value })}
/>
- <IgnoreWorkflowSystemPrompt
- ignoreWorkflowSystemPrompt={data.ignoreWorkflowSystemPrompt ?? false}
- editable={editable}
- onIgnoreWorkflowSystemPromptChange={(ignoreWorkflowSystemPrompt) => {
- update({ ignoreWorkflowSystemPrompt });
- }}
- />
+ <Accordion type="single" collapsible>
+ <AccordionItem value="advanced" className="border-b-0">
+ <AccordionTrigger className="py-0">
+ Advanced Settings
+ </AccordionTrigger>
+ <AccordionContent className="pl-6 pr-1 pt-1">
+ <IgnoreWorkflowSystemPrompt
+ ignoreWorkflowSystemPrompt={
+ data.ignoreWorkflowSystemPrompt ?? false
+ }
+ editable={editable}
+ onIgnoreWorkflowSystemPromptChange={(
+ ignoreWorkflowSystemPrompt,
+ ) => {
+ update({ ignoreWorkflowSystemPrompt });
+ }}
+ />
+ </AccordionContent>
+ </AccordionItem>
+ </Accordion>
</div>
);
}
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/GoogleSheetsWriteNode/validate.ts b/skyvern-frontend/src/routes/workflows/editor/nodes/GoogleSheetsWriteNode/validate.ts
index b9cac7c83..24d0b55dc 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/GoogleSheetsWriteNode/validate.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/GoogleSheetsWriteNode/validate.ts
@@ -1,3 +1,5 @@
+import { getJsonParseErrorDetail } from "@/util/jsonParseError";
+
import type { GoogleSheetsWriteNode } from "./types";
const missing = (value: string): boolean => !value.trim();
@@ -35,8 +37,13 @@ export function validateGoogleSheetsWriteNode(
`${label}: Column mapping must be a JSON object of source field -> column letter.`,
);
}
- } catch {
- errors.push(`${label}: Column mapping is not valid JSON.`);
+ } catch (error) {
+ errors.push(
+ `${label}: Column mapping is not valid JSON: ${getJsonParseErrorDetail(
+ mapping,
+ error,
+ )}`,
+ );
}
}
return errors;
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/HttpUtils.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/HttpUtils.tsx
index c71345fb1..367619e98 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/HttpUtils.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/HttpUtils.tsx
@@ -64,18 +64,18 @@ export function UrlValidator({ url }: { url: string }) {
return (
<div
className={cn(
- "flex items-center gap-1 text-xs",
+ "flex min-w-0 items-start gap-1 text-xs",
validation.valid
? "text-green-600 dark:text-green-400"
: "text-red-600 dark:text-red-400",
)}
>
{validation.valid ? (
- <CheckCircledIcon className="h-3 w-3" />
+ <CheckCircledIcon className="h-3 w-3 shrink-0" />
) : (
- <ExclamationTriangleIcon className="h-3 w-3" />
+ <ExclamationTriangleIcon className="h-3 w-3 shrink-0" />
)}
- <span>{validation.message}</span>
+ <span className="min-w-0 break-words">{validation.message}</span>
</div>
);
}
@@ -89,18 +89,18 @@ export function JsonValidator({ value }: { value: string }) {
return (
<div
className={cn(
- "flex items-center gap-1 text-xs",
+ "flex min-w-0 items-start gap-1 text-xs",
validation.valid
? "text-green-600 dark:text-green-400"
: "text-red-600 dark:text-red-400",
)}
>
{validation.valid ? (
- <CheckCircledIcon className="h-3 w-3" />
+ <CheckCircledIcon className="h-3 w-3 shrink-0" />
) : (
- <ExclamationTriangleIcon className="h-3 w-3" />
+ <ExclamationTriangleIcon className="h-3 w-3 shrink-0" />
)}
- <span>{validation.message}</span>
+ <span className="min-w-0 break-words">{validation.message}</span>
</div>
);
}
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/httpValidation.test.ts b/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/httpValidation.test.ts
index ea6e8521a..ba76a7475 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/httpValidation.test.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/httpValidation.test.ts
@@ -61,6 +61,19 @@ describe("validateJson", () => {
expect(result.valid).toBe(false);
});
+ test("non-breaking spaces outside strings explain the hidden character", () => {
+ const result = validateJson(`{
+ "personalInfo": {
+ "firstName": "Luis",
+ "lastName": "Ortiz"\u00a0
+ }
+}`);
+ expect(result.valid).toBe(false);
+ expect(result.message).toContain(
+ "JSON contains a non-breaking space (U+00A0) at line 4, column 24; replace it with a regular space.",
+ );
+ });
+
test("completely malformed JSON is invalid", () => {
const result = validateJson("{key: value}");
expect(result.valid).toBe(false);
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/httpValidation.ts b/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/httpValidation.ts
index 23c9abd46..d1919560b 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/httpValidation.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/httpValidation.ts
@@ -1,4 +1,5 @@
import { TSON } from "@/util/tson";
+import { getInvalidJsonMessage } from "@/util/jsonParseError";
// URL Validation Helper
export function validateUrl(url: string): { valid: boolean; message: string } {
@@ -31,7 +32,13 @@ export function validateJson(value: string): {
// First check: does it pass TSON (template-aware) parsing?
const tsonResult = TSON.parse(trimmed);
if (!tsonResult.success) {
- return { valid: false, message: tsonResult.error || "Invalid JSON" };
+ return {
+ valid: false,
+ message: getInvalidJsonMessage(
+ trimmed,
+ tsonResult.error || "Parse error",
+ ),
+ };
}
// Second check: does it also pass strict JSON parsing?
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/PDFParserNode/PDFParserEditor.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/PDFParserNode/PDFParserEditor.tsx
index 6aabe3deb..803373f05 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/PDFParserNode/PDFParserEditor.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/PDFParserNode/PDFParserEditor.tsx
@@ -2,6 +2,12 @@ import { useReactFlow } from "@xyflow/react";
import { HelpTooltip } from "@/components/HelpTooltip";
import { ModelSelector } from "@/components/ModelSelector";
+import {
+ Accordion,
+ AccordionContent,
+ AccordionItem,
+ AccordionTrigger,
+} from "@/components/ui/accordion";
import { Label } from "@/components/ui/label";
import { WorkflowBlockInput } from "@/components/WorkflowBlockInput";
import { WorkflowDataSchemaInputGroup } from "@/components/DataSchemaInputGroup/WorkflowDataSchemaInputGroup";
@@ -57,15 +63,26 @@ function PDFParserEditorBody({
value={model}
onChange={(v) => update({ model: v })}
/>
- <IgnoreWorkflowSystemPrompt
- ignoreWorkflowSystemPrompt={
- node.data.ignoreWorkflowSystemPrompt ?? false
- }
- editable={editable}
- onIgnoreWorkflowSystemPromptChange={(ignoreWorkflowSystemPrompt) => {
- update({ ignoreWorkflowSystemPrompt });
- }}
- />
+ <Accordion type="single" collapsible>
+ <AccordionItem value="advanced" className="border-b-0">
+ <AccordionTrigger className="py-0">
+ Advanced Settings
+ </AccordionTrigger>
+ <AccordionContent className="pl-6 pr-1 pt-1">
+ <IgnoreWorkflowSystemPrompt
+ ignoreWorkflowSystemPrompt={
+ node.data.ignoreWorkflowSystemPrompt ?? false
+ }
+ editable={editable}
+ onIgnoreWorkflowSystemPromptChange={(
+ ignoreWorkflowSystemPrompt,
+ ) => {
+ update({ ignoreWorkflowSystemPrompt });
+ }}
+ />
+ </AccordionContent>
+ </AccordionItem>
+ </Accordion>
</div>
);
}
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/PdfFillNode/PdfFillEditor.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/PdfFillNode/PdfFillEditor.tsx
index 1cc071b35..f723dca0b 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/PdfFillNode/PdfFillEditor.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/PdfFillNode/PdfFillEditor.tsx
@@ -152,13 +152,6 @@ function PdfFillEditorBody({
value={data.model}
onChange={(value) => update({ model: value })}
/>
- <IgnoreWorkflowSystemPrompt
- ignoreWorkflowSystemPrompt={data.ignoreWorkflowSystemPrompt ?? false}
- editable={editable}
- onIgnoreWorkflowSystemPromptChange={(ignoreWorkflowSystemPrompt) => {
- update({ ignoreWorkflowSystemPrompt });
- }}
- />
<Accordion type="single" collapsible>
<AccordionItem value="advanced" className="border-b-0">
@@ -174,6 +167,17 @@ function PdfFillEditorBody({
update({ parameterKeys });
}}
/>
+ <IgnoreWorkflowSystemPrompt
+ ignoreWorkflowSystemPrompt={
+ data.ignoreWorkflowSystemPrompt ?? false
+ }
+ editable={editable}
+ onIgnoreWorkflowSystemPromptChange={(
+ ignoreWorkflowSystemPrompt,
+ ) => {
+ update({ ignoreWorkflowSystemPrompt });
+ }}
+ />
<div className="space-y-2">
<div className="flex gap-2">
<Label className="text-xs text-slate-300">LLM Key</Label>
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptEditor.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptEditor.tsx
index c5aedccc4..fff285d4c 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptEditor.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptEditor.tsx
@@ -2,6 +2,12 @@ import { useReactFlow } from "@xyflow/react";
import { HelpTooltip } from "@/components/HelpTooltip";
import { ModelSelector } from "@/components/ModelSelector";
+import {
+ Accordion,
+ AccordionContent,
+ AccordionItem,
+ AccordionTrigger,
+} from "@/components/ui/accordion";
import { Label } from "@/components/ui/label";
import { Separator } from "@/components/ui/separator";
import { WorkflowBlockInputTextarea } from "@/components/WorkflowBlockInputTextarea";
@@ -66,15 +72,26 @@ function TextPromptEditorBody({
onChange={(next) => update({ jsonSchema: next })}
suggestionContext={{ current_schema: jsonSchema }}
/>
- <IgnoreWorkflowSystemPrompt
- ignoreWorkflowSystemPrompt={
- node.data.ignoreWorkflowSystemPrompt ?? false
- }
- editable={editable}
- onIgnoreWorkflowSystemPromptChange={(ignoreWorkflowSystemPrompt) => {
- update({ ignoreWorkflowSystemPrompt });
- }}
- />
+ <Accordion type="single" collapsible>
+ <AccordionItem value="advanced" className="border-b-0">
+ <AccordionTrigger className="py-0">
+ Advanced Settings
+ </AccordionTrigger>
+ <AccordionContent className="pl-6 pr-1 pt-1">
+ <IgnoreWorkflowSystemPrompt
+ ignoreWorkflowSystemPrompt={
+ node.data.ignoreWorkflowSystemPrompt ?? false
+ }
+ editable={editable}
+ onIgnoreWorkflowSystemPromptChange={(
+ ignoreWorkflowSystemPrompt,
+ ) => {
+ update({ ignoreWorkflowSystemPrompt });
+ }}
+ />
+ </AccordionContent>
+ </AccordionItem>
+ </Accordion>
</div>
);
}
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/components/BlockParametersDialog.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/components/BlockParametersDialog.tsx
index fa262f7ae..8aa622c6d 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/components/BlockParametersDialog.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/components/BlockParametersDialog.tsx
@@ -14,6 +14,7 @@ import { ScrollArea, ScrollAreaViewport } from "@/components/ui/scroll-area";
import { WorkflowParameterInput } from "@/routes/workflows/WorkflowParameterInput";
import { getLabelForWorkflowParameterType } from "@/routes/workflows/editor/workflowEditorUtils";
import type { WorkflowParameter } from "@/routes/workflows/types/workflowTypes";
+import { getInvalidJsonMessage } from "@/util/jsonParseError";
/**
* Validates a parameter value based on its type.
@@ -34,8 +35,7 @@ function validateParameterValue(value: unknown, type: string): string | null {
JSON.parse(trimmed);
return null;
} catch (e) {
- const message = e instanceof SyntaxError ? e.message : "Parse error";
- return `Invalid JSON: ${message}`;
+ return getInvalidJsonMessage(trimmed, e);
}
}
return null;
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx
index 36928acbb..ee3b54f95 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx
@@ -66,6 +66,7 @@ import {
useWorkflowSettingsStore,
type WorkflowSettingsState,
} from "@/store/WorkflowSettingsStore";
+import { getJsonParseErrorDetail } from "@/util/jsonParseError";
import { cn, formatDate, toDate } from "@/util/utils";
import {
statusIsAFailureType,
@@ -188,7 +189,10 @@ const getPayload = (opts: {
toast({
variant: "warning",
title: "Extra HTTP Headers",
- description: "Invalid extra HTTP Headers JSON",
+ description: `Invalid extra HTTP Headers JSON: ${getJsonParseErrorDetail(
+ String(opts.workflowSettings.extraHttpHeaders ?? ""),
+ e,
+ )}`,
});
}
diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/FileParserBlockForm.test.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/FileParserBlockForm.test.tsx
index f3d553d51..211fda8b9 100644
--- a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/FileParserBlockForm.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/FileParserBlockForm.test.tsx
@@ -159,6 +159,13 @@ describe("FileParserBlockForm (SKY-9381)", () => {
expect(screen.getByText("Auto detect")).toBeDefined();
expect(screen.getByTestId("json-schema-input")).toBeDefined();
expect(screen.getByTestId("model-selector")).toBeDefined();
+ const advancedSettings = screen.getByText("Advanced Settings");
+ fireEvent.click(advancedSettings);
+ expect(
+ advancedSettings.compareDocumentPosition(
+ screen.getByText("Ignore System Prompt"),
+ ) & Node.DOCUMENT_POSITION_FOLLOWING,
+ ).toBeTruthy();
});
test("File URL onChange dispatches updateNodeData via useUpdate (byte-identical write)", () => {
diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/PDFParserBlockForm.test.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/PDFParserBlockForm.test.tsx
index 3ab3b1d61..f950cb8b5 100644
--- a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/PDFParserBlockForm.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/PDFParserBlockForm.test.tsx
@@ -154,6 +154,13 @@ describe("PDFParserBlockForm (SKY-93XX)", () => {
expect(
(screen.getByTestId("model-select") as HTMLSelectElement).value,
).toBe("gpt-4o");
+ const advancedSettings = screen.getByText("Advanced Settings");
+ fireEvent.click(advancedSettings);
+ expect(
+ advancedSettings.compareDocumentPosition(
+ screen.getByText("Ignore System Prompt"),
+ ) & Node.DOCUMENT_POSITION_FOLLOWING,
+ ).toBeTruthy();
});
test("editing fileUrl propagates via updateNodeData", () => {
diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/PdfFillBlockForm.test.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/PdfFillBlockForm.test.tsx
index 8157ad71e..98d0e6960 100644
--- a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/PdfFillBlockForm.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/PdfFillBlockForm.test.tsx
@@ -217,6 +217,13 @@ describe("PdfFillBlockForm", () => {
expect(screen.getByText("Prompt")).toBeDefined();
expect(screen.getByText("Payload")).toBeDefined();
expect(screen.queryByText("API Key")).toBeNull();
+ expect(
+ screen
+ .getByText("Advanced Settings")
+ .compareDocumentPosition(
+ screen.getByTestId("ignore-workflow-system-prompt"),
+ ) & Node.DOCUMENT_POSITION_FOLLOWING,
+ ).toBeTruthy();
});
test("editing file URL, prompt, and payload propagates", () => {
diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/TextPromptBlockForm.test.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/TextPromptBlockForm.test.tsx
index 9ea231287..7e0a83c2d 100644
--- a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/TextPromptBlockForm.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/TextPromptBlockForm.test.tsx
@@ -158,6 +158,13 @@ describe("TextPromptBlockForm (SKY-9378)", () => {
expect(
screen.getByTestId("model-selector").getAttribute("data-value"),
).toBe("gpt-4");
+ const advancedSettings = screen.getByText("Advanced Settings");
+ fireEvent.click(advancedSettings);
+ expect(
+ advancedSettings.compareDocumentPosition(
+ screen.getByText("Ignore System Prompt"),
+ ) & Node.DOCUMENT_POSITION_FOLLOWING,
+ ).toBeTruthy();
});
test("persists prompt edits immediately via updateNodeData", () => {
diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowParameterEditPanel.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowParameterEditPanel.tsx
index 33c07bb55..54527490a 100644
--- a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowParameterEditPanel.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowParameterEditPanel.tsx
@@ -38,6 +38,7 @@ import { getDefaultValueForParameterType } from "../workflowEditorUtils";
import { validateBitwardenLoginCredential } from "./util";
import { HelpTooltip } from "@/components/HelpTooltip";
import { useCustomCredentialServiceConfig } from "@/hooks/useCustomCredentialServiceConfig";
+import { getInvalidJsonMessage } from "@/util/jsonParseError";
import {
CredentialDataType,
CredentialSource,
@@ -1040,32 +1041,28 @@ function WorkflowParameterEditPanel({
// Handle workflow parameters (non-credential)
if (type === "workflow" && !isCredentialSelected) {
+ let defaultValue = defaultValueState.defaultValue;
+
if (
parameterType === "json" &&
typeof defaultValueState.defaultValue === "string"
) {
try {
- JSON.parse(defaultValueState.defaultValue);
+ defaultValue = JSON.parse(defaultValueState.defaultValue);
} catch (e) {
toast({
variant: "destructive",
title: "Failed to save input",
- description: "Invalid JSON for default value",
+ description: getInvalidJsonMessage(
+ defaultValueState.defaultValue,
+ e,
+ ),
});
return;
}
}
- let defaultValue = defaultValueState.defaultValue;
-
- // Handle JSON parsing
- if (
- parameterType === "json" &&
- typeof defaultValueState.defaultValue === "string"
- ) {
- defaultValue = JSON.parse(defaultValueState.defaultValue);
- }
// Convert boolean to string for backend storage
- else if (
+ if (
parameterType === "boolean" &&
typeof defaultValueState.defaultValue === "boolean"
) {
diff --git a/skyvern-frontend/src/routes/workflows/editor/validateErrorCodeMapping.test.ts b/skyvern-frontend/src/routes/workflows/editor/validateErrorCodeMapping.test.ts
index 6e623adbc..2e88bc3ed 100644
--- a/skyvern-frontend/src/routes/workflows/editor/validateErrorCodeMapping.test.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/validateErrorCodeMapping.test.ts
@@ -13,7 +13,7 @@ describe("validateErrorCodeMapping", () => {
test("invalid JSON returns a parse error", () => {
const result = validateErrorCodeMapping("block_1", "{not json");
expect(result).toHaveLength(1);
- expect(result[0]).toContain("Error messages is not valid JSON");
+ expect(result[0]).toContain("Error messages are not valid JSON");
});
test("null value (mapping disabled) returns no errors", () => {
diff --git a/skyvern-frontend/src/routes/workflows/editor/validateErrorCodeMapping.ts b/skyvern-frontend/src/routes/workflows/editor/validateErrorCodeMapping.ts
index b30ef85c2..c750eab72 100644
--- a/skyvern-frontend/src/routes/workflows/editor/validateErrorCodeMapping.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/validateErrorCodeMapping.ts
@@ -1,3 +1,5 @@
+import { getJsonParseErrorDetail } from "@/util/jsonParseError";
+
/**
* Validate an errorCodeMapping JSON string. The value must:
* - parse as JSON,
@@ -17,8 +19,13 @@ export function validateErrorCodeMapping(
let parsed: unknown;
try {
parsed = JSON.parse(errorCodeMapping);
- } catch {
- errors.push(`${label}: Error messages is not valid JSON.`);
+ } catch (error) {
+ errors.push(
+ `${label}: Error messages are not valid JSON: ${getJsonParseErrorDetail(
+ errorCodeMapping,
+ error,
+ )}`,
+ );
return errors;
}
// `null` is the disabled sentinel — valid.
diff --git a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts
index 3888d27f5..bec9969fe 100644
--- a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts
@@ -3,6 +3,7 @@ import { type Node, Edge } from "@xyflow/react";
import { nanoid } from "nanoid";
import { TSON } from "@/util/tson";
+import { getJsonParseErrorDetail } from "@/util/jsonParseError";
import {
WorkflowBlockType,
@@ -4603,7 +4604,10 @@ function getWorkflowErrors(nodes: Array<AppNode>): Array<string> {
if (!result.success) {
errors.push(
- `${node.data.label}: Data schema has invalid templated JSON: ${result.error ?? "-"}`,
+ `${node.data.label}: Data schema has invalid templated JSON: ${getJsonParseErrorDetail(
+ node.data.dataSchema,
+ result.error ?? "Parse error",
+ )}`,
);
}
}
@@ -4689,7 +4693,10 @@ function getWorkflowErrors(nodes: Array<AppNode>): Array<string> {
if (!result.success) {
errors.push(
- `${node.data.label}: Data schema has invalid templated JSON: ${result.error ?? "-"}`,
+ `${node.data.label}: Data schema has invalid templated JSON: ${getJsonParseErrorDetail(
+ node.data.dataSchema,
+ result.error ?? "Parse error",
+ )}`,
);
}
}
@@ -4699,8 +4706,13 @@ function getWorkflowErrors(nodes: Array<AppNode>): Array<string> {
textPromptNodes.forEach((node) => {
try {
JSON.parse(node.data.jsonSchema);
- } catch {
- errors.push(`${node.data.label}: Data schema is not valid JSON.`);
+ } catch (error) {
+ errors.push(
+ `${node.data.label}: Data schema is not valid JSON: ${getJsonParseErrorDetail(
+ node.data.jsonSchema,
+ error,
+ )}`,
+ );
}
});
@@ -4708,8 +4720,13 @@ function getWorkflowErrors(nodes: Array<AppNode>): Array<string> {
pdfParserNodes.forEach((node) => {
try {
JSON.parse(node.data.jsonSchema);
- } catch {
- errors.push(`${node.data.label}: Data schema is not valid JSON.`);
+ } catch (error) {
+ errors.push(
+ `${node.data.label}: Data schema is not valid JSON: ${getJsonParseErrorDetail(
+ node.data.jsonSchema,
+ error,
+ )}`,
+ );
}
});
@@ -4717,8 +4734,13 @@ function getWorkflowErrors(nodes: Array<AppNode>): Array<string> {
fileParserNodes.forEach((node) => {
try {
JSON.parse(node.data.jsonSchema);
- } catch {
- errors.push(`${node.data.label}: Data schema is not valid JSON.`);
+ } catch (error) {
+ errors.push(
+ `${node.data.label}: Data schema is not valid JSON: ${getJsonParseErrorDetail(
+ node.data.jsonSchema,
+ error,
+ )}`,
+ );
}
});
diff --git a/skyvern-frontend/src/store/WorkflowHasChangesStore.ts b/skyvern-frontend/src/store/WorkflowHasChangesStore.ts
index bdcef7a87..c6bd85837 100644
--- a/skyvern-frontend/src/store/WorkflowHasChangesStore.ts
+++ b/skyvern-frontend/src/store/WorkflowHasChangesStore.ts
@@ -8,6 +8,7 @@ import { usePostHog } from "posthog-js/react";
import { getClient } from "@/api/AxiosClient";
import { toast } from "@/components/ui/use-toast";
import { useCredentialGetter } from "@/hooks/useCredentialGetter";
+import { getJsonParseErrorDetail } from "@/util/jsonParseError";
import {
type BlockYAML,
type ParameterYAML,
@@ -131,7 +132,10 @@ const useWorkflowSave = (opts?: WorkflowSaveOpts) => {
} catch (error) {
toast({
title: "Error",
- description: "Invalid JSON format in extra http headers",
+ description: `Invalid JSON format in extra http headers: ${getJsonParseErrorDetail(
+ saveData.settings.extraHttpHeaders ?? "",
+ error,
+ )}`,
variant: "destructive",
});
return;
@@ -163,7 +167,10 @@ const useWorkflowSave = (opts?: WorkflowSaveOpts) => {
} catch (error) {
toast({
title: "Error",
- description: "Invalid JSON format in cdp connect headers",
+ description: `Invalid JSON format in cdp connect headers: ${getJsonParseErrorDetail(
+ saveData.settings.cdpConnectHeaders ?? "",
+ error,
+ )}`,
variant: "destructive",
});
return;
diff --git a/skyvern-frontend/src/util/jsonParseError.test.ts b/skyvern-frontend/src/util/jsonParseError.test.ts
new file mode 100644
index 000000000..f175f8892
--- /dev/null
+++ b/skyvern-frontend/src/util/jsonParseError.test.ts
@@ -0,0 +1,60 @@
+import { describe, expect, test } from "vitest";
+
+import {
+ getInvalidJsonMessage,
+ getJsonParseErrorDetail,
+} from "./jsonParseError";
+
+describe("json parse error formatting", () => {
+ test("explains hidden non-breaking spaces outside strings", () => {
+ const value = `{
+ "personalInfo": {
+ "firstName": "Luis",
+ "lastName": "Ortiz"\u00a0
+ }
+}`;
+ let error: unknown;
+
+ try {
+ JSON.parse(value);
+ } catch (caught) {
+ error = caught;
+ }
+
+ expect(getInvalidJsonMessage(value, error)).toContain(
+ "Invalid JSON: JSON contains a non-breaking space (U+00A0) at line 4, column 24; replace it with a regular space.",
+ );
+ });
+
+ test("preserves parser details for ordinary JSON syntax errors", () => {
+ const value = '{"first": "Luis",}';
+ let error: unknown;
+
+ try {
+ JSON.parse(value);
+ } catch (caught) {
+ error = caught;
+ }
+
+ expect(getJsonParseErrorDetail(value, error)).toMatch(
+ /Expected double-quoted property name|Unexpected token/,
+ );
+ });
+
+ test("adds line and column when the parser only reports a position", () => {
+ const value = '{\n "first": "Luis",\n}';
+
+ expect(
+ getJsonParseErrorDetail(
+ value,
+ "Unexpected token } in JSON at position 21",
+ ),
+ ).toBe("Unexpected token } in JSON at position 21 (line 3 column 1)");
+ });
+
+ test("does not flag non-breaking spaces inside strings", () => {
+ const value = '{"lastName": "Ortiz\u00a0"}';
+
+ expect(getJsonParseErrorDetail(value, "Parse error")).toBe("Parse error");
+ });
+});
diff --git a/skyvern-frontend/src/util/jsonParseError.ts b/skyvern-frontend/src/util/jsonParseError.ts
new file mode 100644
index 000000000..c4a67303d
--- /dev/null
+++ b/skyvern-frontend/src/util/jsonParseError.ts
@@ -0,0 +1,133 @@
+type JsonLocation = {
+ position: number;
+ line: number;
+ column: number;
+};
+
+const allowedJsonWhitespace = new Set([" ", "\n", "\r", "\t"]);
+
+const unsupportedWhitespaceNames: Record<string, string> = {
+ "\u00a0": "non-breaking space",
+ "\u1680": "ogham space mark",
+ "\u2000": "en quad",
+ "\u2001": "em quad",
+ "\u2002": "en space",
+ "\u2003": "em space",
+ "\u2004": "three-per-em space",
+ "\u2005": "four-per-em space",
+ "\u2006": "six-per-em space",
+ "\u2007": "figure space",
+ "\u2008": "punctuation space",
+ "\u2009": "thin space",
+ "\u200a": "hair space",
+ "\u2028": "line separator",
+ "\u2029": "paragraph separator",
+ "\u202f": "narrow non-breaking space",
+ "\u205f": "medium mathematical space",
+ "\u3000": "ideographic space",
+ "\ufeff": "byte order mark",
+};
+
+function getLocation(input: string, position: number): JsonLocation {
+ let line = 1;
+ let column = 1;
+
+ for (let i = 0; i < position; i++) {
+ if (input.charAt(i) === "\n") {
+ line++;
+ column = 1;
+ } else {
+ column++;
+ }
+ }
+
+ return { position, line, column };
+}
+
+function getCodePointLabel(char: string): string {
+ const codePoint = char.codePointAt(0) ?? char.charCodeAt(0);
+ return `U+${codePoint.toString(16).toUpperCase().padStart(4, "0")}`;
+}
+
+function getErrorMessage(error: unknown): string {
+ if (error instanceof Error) {
+ return error.message;
+ }
+ if (typeof error === "string") {
+ return error;
+ }
+ return String(error);
+}
+
+function getUnsupportedWhitespace(input: string): {
+ char: string;
+ location: JsonLocation;
+} | null {
+ let inString = false;
+ let escapeNext = false;
+
+ for (let i = 0; i < input.length; i++) {
+ const char = input.charAt(i);
+
+ if (escapeNext) {
+ escapeNext = false;
+ continue;
+ }
+
+ if (char === "\\") {
+ escapeNext = true;
+ continue;
+ }
+
+ if (char === '"') {
+ inString = !inString;
+ continue;
+ }
+
+ if (!inString && !allowedJsonWhitespace.has(char) && /\s/u.test(char)) {
+ return { char, location: getLocation(input, i) };
+ }
+ }
+
+ return null;
+}
+
+function addLocationFromPosition(input: string, message: string): string {
+ if (/line \d+ column \d+/i.test(message)) {
+ return message;
+ }
+
+ const positionMatch = message.match(/position (\d+)/i);
+ if (!positionMatch) {
+ return message;
+ }
+
+ const position = Number(positionMatch[1]);
+ if (!Number.isFinite(position)) {
+ return message;
+ }
+
+ const { line, column } = getLocation(input, position);
+ return `${message} (line ${line} column ${column})`;
+}
+
+export function getJsonParseErrorDetail(input: string, error: unknown): string {
+ const message = getErrorMessage(error) || "Parse error";
+ const unsupportedWhitespace = getUnsupportedWhitespace(input);
+
+ if (unsupportedWhitespace) {
+ const name =
+ unsupportedWhitespaceNames[unsupportedWhitespace.char] ??
+ "unsupported whitespace";
+ const codePoint = getCodePointLabel(unsupportedWhitespace.char);
+ const { line, column } = unsupportedWhitespace.location;
+
+ return `JSON contains a ${name} (${codePoint}) at line ${line}, column ${column}; replace it with a regular space. Parser detail: ${addLocationFromPosition(input, message)}`;
+ }
+
+ return addLocationFromPosition(input, message);
+}
+
+export function getInvalidJsonMessage(input: string, error: unknown): string {
+ return `Invalid JSON: ${getJsonParseErrorDetail(input, error)}`;
+}
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index f3018fb0e..b011fb000 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -33,6 +33,8 @@ from charset_normalizer import from_bytes
from email_validator import EmailNotValidError, validate_email
from jinja2 import StrictUndefined, TemplateSyntaxError
from jinja2.sandbox import SandboxedEnvironment
+from jsonschema import Draft202012Validator
+from jsonschema.exceptions import ValidationError
from opentelemetry import trace as otel_trace
from playwright.async_api import Page
from pydantic import BaseModel, Field, model_validator
@@ -83,6 +85,8 @@ from skyvern.forge.sdk.api.files import (
from skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandler
from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory
from skyvern.forge.sdk.api.llm.custom_llm_registry import is_custom_llm_model_name
+from skyvern.forge.sdk.api.llm.exceptions import InvalidLLMResponseFormat, InvalidLLMResponseType
+from skyvern.forge.sdk.api.llm.schema_validator import validate_schema
from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.aiohttp_helper import aiohttp_request
@@ -4378,6 +4382,173 @@ async def wrapper({default_args}):
)
+SCHEMA_VALIDATION_MAX_ATTEMPTS = 2
+SCHEMA_VALIDATION_MAX_ERRORS = 5
+
+
+def _default_structured_output_schema(description: str) -> dict[str, Any]:
+ # The output field is optional to preserve the legacy permissive default schema.
+ return {
+ "type": "object",
+ "properties": {
+ "output": {
+ "type": "object",
+ "description": description,
+ }
+ },
+ }
+
+
+def _default_text_prompt_schema() -> dict[str, Any]:
+ return {
+ "type": "object",
+ "properties": {
+ "llm_response": {
+ "type": "string",
+ "description": "Your response to the prompt",
+ }
+ },
+ }
+
+
+def _json_type_name(value: Any) -> str:
+ if value is None:
+ return "null"
+ if isinstance(value, bool):
+ return "boolean"
+ if isinstance(value, dict):
+ return "object"
+ if isinstance(value, list):
+ return "array"
+ if isinstance(value, str):
+ return "string"
+ if isinstance(value, int):
+ return "integer"
+ if isinstance(value, float):
+ return "number"
+ return type(value).__name__
+
+
+def _schema_type_description(schema_type: Any) -> str:
+ if isinstance(schema_type, list):
+ return " or ".join(str(t) for t in schema_type)
+ return str(schema_type)
+
+
+def _schema_path(error: ValidationError) -> str:
+ schema_path = list(error.absolute_schema_path)
+ path_parts: list[str] = []
+ index = 0
+ while index < len(schema_path):
+ part = schema_path[index]
+ if part == "properties" and index + 1 < len(schema_path):
+ path_parts.append(str(schema_path[index + 1]))
+ index += 2
+ continue
+ if part == "items":
+ path_parts.append("[]")
+ index += 1
+ continue
+ if part == "additionalProperties":
+ path_parts.append("<map value>")
+ index += 1
+ continue
+ if part == "patternProperties":
+ path_parts.append("<map value>")
+ index += 2 if index + 1 < len(schema_path) else 1
+ continue
+ index += 1
+
+ return "root" + "".join(f"{part}" if part == "[]" else f".{part}" for part in path_parts)
+
+
+def _format_schema_validation_error(error: ValidationError) -> str:
+ path = _schema_path(error)
+ actual_type = _json_type_name(error.instance)
+
+ if error.validator == "type":
+ expected_type = _schema_type_description(error.validator_value)
+ return f"{path}: expected type {expected_type}, got {actual_type}"
+
+ if error.validator == "required":
+ match = re.match(r"'([^']+)' is a required property", error.message)
+ if match:
+ return f"{path}: missing required property {match.group(1)}"
+ return f"{path}: missing required property"
+
+ if error.validator == "additionalProperties":
+ unexpected_count: int | None = None
+ schema_properties = error.schema.get("properties", {}) if isinstance(error.schema, dict) else {}
+ if isinstance(error.instance, dict) and isinstance(schema_properties, dict):
+ unexpected_count = sum(1 for field in error.instance if field not in schema_properties)
+ if unexpected_count is not None:
+ return f"{path}: has {unexpected_count} unexpected properties"
+ return f"{path}: has unexpected properties"
+
+ if error.validator in {"minItems", "maxItems"} and isinstance(error.instance, list):
+ return f"{path}: violates {error.validator}={error.validator_value}; item count={len(error.instance)}"
+
+ if error.validator in {"minLength", "maxLength"} and isinstance(error.instance, str):
+ return f"{path}: violates {error.validator}={error.validator_value}; string length={len(error.instance)}"
+
+ if error.validator == "enum":
+ allowed_count = len(error.validator_value) if isinstance(error.validator_value, list) else "configured"
+ return f"{path}: value is not one of {allowed_count} allowed values; got {actual_type}"
+
+ return f"{path}: violates {error.validator} constraint; got {actual_type}"
+
+
+def _validate_response_against_json_schema(
+ response: Any,
+ json_schema: dict[str, Any] | None,
+ schema_label: str,
+ max_errors: int = SCHEMA_VALIDATION_MAX_ERRORS,
+) -> str | None:
+ if not json_schema:
+ return None
+
+ if not validate_schema(json_schema):
+ return f"{schema_label} JSON schema is invalid."
+
+ try:
+ validator = Draft202012Validator(json_schema)
+ validation_errors = [_format_schema_validation_error(error) for error in validator.iter_errors(response)]
+ except Exception as e:
+ LOG.warning(
+ "Failed to validate LLM response against JSON schema",
+ schema_label=schema_label,
+ error_type=type(e).__name__,
+ exc_info=True,
+ )
+ return f"{schema_label} JSON schema validation failed ({type(e).__name__})."
+
+ validation_errors = list(dict.fromkeys(validation_errors))
+ if not validation_errors:
+ return None
+
+ return f"LLM response does not match {schema_label.lower()} JSON schema: " + "; ".join(
+ validation_errors[:max_errors]
+ )
+
+
+def _is_schema_configuration_failure(failure_reason: str) -> bool:
+ return "JSON schema is invalid" in failure_reason or "JSON schema validation failed" in failure_reason
+
+
+def _llm_response_format_failure_reason(error: Exception) -> str:
+ return f"LLM response could not be parsed or coerced into the required JSON shape ({type(error).__name__})."
+
+
+def _build_schema_validation_retry_prompt(prompt: str, failure_reason: str) -> str:
+ return (
+ f"{prompt}\n\n"
+ "Your previous response failed JSON schema validation.\n"
+ f"Validation error: {failure_reason}\n\n"
+ "Retry the task. Return only valid JSON that exactly matches the schema. "
+ "Do not include markdown, code fences, explanatory text, or extra fields."
+ )
+
+
class TextPromptBlock(Block):
# There is a mypy bug with Literal. Without the type: ignore, mypy will raise an error:
# Parameter 1 of Literal[...] cannot be of type "Any"
@@ -4387,6 +4558,8 @@ class TextPromptBlock(Block):
prompt: str
parameters: list[PARAMETER_TYPE] = []
json_schema: dict[str, Any] | None = None
+ schema_validation_max_attempts: ClassVar[int] = SCHEMA_VALIDATION_MAX_ATTEMPTS
+ schema_validation_max_errors: ClassVar[int] = SCHEMA_VALIDATION_MAX_ERRORS
def get_all_parameters(
self,
@@ -4423,6 +4596,14 @@ class TextPromptBlock(Block):
self._apply_workflow_system_prompt(workflow_run_context)
+ def _validate_response_against_json_schema(self, response: Any) -> str | None:
+ return _validate_response_against_json_schema(
+ response,
+ self.json_schema,
+ "Text prompt",
+ max_errors=self.schema_validation_max_errors,
+ )
+
async def send_prompt(
self,
prompt: str,
@@ -4430,28 +4611,23 @@ class TextPromptBlock(Block):
workflow_run_id: str,
organization_id: str | None = None,
workflow_run_block_id: str | None = None,
- ) -> dict[str, Any]:
+ schema_validation_failure: str | None = None,
+ json_schema: dict[str, Any] | None = None,
+ ) -> dict[str, Any] | list | str | None:
default_llm_handler = await self._resolve_default_llm_handler(workflow_run_id, organization_id)
llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(
self.override_llm_key_for_organization(organization_id) or self.llm_key, default=default_llm_handler
)
- if not self.json_schema:
- self.json_schema = {
- "type": "object",
- "properties": {
- "llm_response": {
- "type": "string",
- "description": "Your response to the prompt",
- }
- },
- }
+ schema_to_use = json_schema or self.json_schema or _default_text_prompt_schema()
prompt = prompt_engine.load_prompt_from_string(prompt, **parameter_values)
+ if schema_validation_failure:
+ prompt = _build_schema_validation_retry_prompt(prompt, schema_validation_failure)
prompt += (
"\n\n"
+ "Please respond to the prompt above using the following JSON definition:\n\n"
+ "```json\n"
- + json.dumps(self.json_schema, indent=2)
+ + json.dumps(schema_to_use, indent=2)
+ "\n```\n\n"
)
@@ -4478,6 +4654,8 @@ class TextPromptBlock(Block):
system_prompt=self.workflow_system_prompt,
workflow_run_block_id=workflow_run_block_id,
organization_id=organization_id,
+ # Schema validation must inspect the raw parsed root; dict coercion can hide wrong-root responses.
+ force_dict=False,
)
if workflow_run_block:
@@ -4570,33 +4748,111 @@ class TextPromptBlock(Block):
else:
parameter_values[parameter.key] = value
- try:
- response = await self.send_prompt(
- self.prompt,
- parameter_values,
- workflow_run_id,
- organization_id,
- workflow_run_block_id=workflow_run_block_id,
- )
- except Exception as e:
- try:
- resolved_llm_key = self.override_llm_key_for_organization(organization_id) or self.llm_key
- except Exception:
- resolved_llm_key = self.llm_key
- LOG.exception(
- "TextPromptBlock LLM call failed",
- block_label=self.label,
- workflow_run_id=workflow_run_id,
- llm_key=resolved_llm_key,
- )
+ response: dict[str, Any] | list | str | None = None
+ schema_to_use = self.json_schema or _default_text_prompt_schema()
+ if not validate_schema(schema_to_use):
return await self.build_block_result(
success=False,
- failure_reason=f"LLM call failed: {e}",
+ failure_reason="Text prompt JSON schema is invalid.",
output_parameter_value=None,
status=BlockStatus.failed,
workflow_run_block_id=workflow_run_block_id,
organization_id=organization_id,
)
+
+ schema_validation_failure_for_retry: str | None = None
+ for attempt in range(self.schema_validation_max_attempts):
+ try:
+ response = await self.send_prompt(
+ self.prompt,
+ parameter_values,
+ workflow_run_id,
+ organization_id,
+ workflow_run_block_id=workflow_run_block_id,
+ schema_validation_failure=schema_validation_failure_for_retry,
+ json_schema=schema_to_use,
+ )
+ except (InvalidLLMResponseFormat, InvalidLLMResponseType) as e:
+ response_format_failure_reason = _llm_response_format_failure_reason(e)
+ will_retry = attempt + 1 < self.schema_validation_max_attempts
+ LOG.warning(
+ "TextPromptBlock LLM response failed response-format validation",
+ block_label=self.label,
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ attempt=attempt + 1,
+ max_attempts=self.schema_validation_max_attempts,
+ will_retry=will_retry,
+ error_type=type(e).__name__,
+ schema_type=schema_to_use.get("type"),
+ )
+ if not will_retry:
+ return await self.build_block_result(
+ success=False,
+ failure_reason=response_format_failure_reason,
+ output_parameter_value=None,
+ status=BlockStatus.failed,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
+
+ schema_validation_failure_for_retry = response_format_failure_reason
+ continue
+ except Exception as e:
+ try:
+ resolved_llm_key = self.override_llm_key_for_organization(organization_id) or self.llm_key
+ except Exception:
+ resolved_llm_key = self.llm_key
+ LOG.exception(
+ "TextPromptBlock LLM call failed",
+ block_label=self.label,
+ workflow_run_id=workflow_run_id,
+ llm_key=resolved_llm_key,
+ )
+ return await self.build_block_result(
+ success=False,
+ failure_reason=f"LLM call failed: {e}",
+ output_parameter_value=None,
+ status=BlockStatus.failed,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
+
+ schema_validation_failure = _validate_response_against_json_schema(
+ response,
+ schema_to_use,
+ "Text prompt",
+ max_errors=self.schema_validation_max_errors,
+ )
+ if not schema_validation_failure:
+ break
+
+ is_schema_configuration_failure = _is_schema_configuration_failure(schema_validation_failure)
+ will_retry = attempt + 1 < self.schema_validation_max_attempts and not is_schema_configuration_failure
+ LOG.warning(
+ "TextPromptBlock LLM response failed schema validation",
+ block_label=self.label,
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ attempt=attempt + 1,
+ max_attempts=self.schema_validation_max_attempts,
+ will_retry=will_retry,
+ failure_reason=schema_validation_failure,
+ schema_type=schema_to_use.get("type"),
+ )
+ if not will_retry:
+ return await self.build_block_result(
+ success=False,
+ failure_reason=schema_validation_failure,
+ output_parameter_value=None,
+ status=BlockStatus.failed,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
+
+ schema_validation_failure_for_retry = schema_validation_failure
+ continue
+
await self.record_output_parameter_value(workflow_run_context, workflow_run_id, response)
return await self.build_block_result(
success=True,
@@ -5789,6 +6045,8 @@ class FileParserBlock(Block):
file_url: str
file_type: FileType = FileType.AUTO_DETECT
json_schema: dict[str, Any] | None = None
+ schema_validation_max_attempts: ClassVar[int] = SCHEMA_VALIDATION_MAX_ATTEMPTS
+ ocr_validation_max_attempts: ClassVar[int] = SCHEMA_VALIDATION_MAX_ATTEMPTS
def get_failure_error_codes(self) -> list[str]:
return ["FILE_PARSER_ERROR"]
@@ -5809,6 +6067,38 @@ class FileParserBlock(Block):
self._apply_workflow_system_prompt(workflow_run_context)
+ @staticmethod
+ def _validate_ocr_llm_response(llm_response: Any) -> str | None:
+ if not isinstance(llm_response, dict):
+ return (
+ f"OCR response must be a JSON object with extracted_text string; got {_json_type_name(llm_response)}."
+ )
+ if not isinstance(llm_response.get("extracted_text"), str):
+ return (
+ "OCR response must include extracted_text as a string; "
+ f"got {_json_type_name(llm_response.get('extracted_text'))}."
+ )
+ return None
+
+ @staticmethod
+ def _build_ocr_validation_retry_prompt(prompt: str, failure_reason: str) -> str:
+ return (
+ f"{prompt}\n\n"
+ "Your previous OCR response failed JSON validation.\n"
+ f"Validation error: {failure_reason}\n\n"
+ 'Retry the task. Return only valid JSON with this exact shape: {"extracted_text": "..."} '
+ "Do not include markdown, code fences, explanatory text, or extra fields."
+ )
+
+ @staticmethod
+ def _validate_ai_response_against_json_schema(response: Any, json_schema: dict[str, Any]) -> str | None:
+ return _validate_response_against_json_schema(
+ response,
+ json_schema,
+ "File parser",
+ max_errors=SCHEMA_VALIDATION_MAX_ERRORS,
+ )
+
def _detect_file_type_from_url(self, file_url: str, file_path: str | None = None) -> FileType:
"""Detect file type based on file extension in the URL, with magic-byte fallback."""
url_parsed = urlparse(file_url)
@@ -6089,6 +6379,9 @@ class FileParserBlock(Block):
bounded concurrency, reassembled in page order with page markers, and
truncated at a page boundary once MAX_FILE_PARSE_INPUT_TOKENS is reached.
"""
+ if self.ocr_validation_max_attempts <= 0:
+ raise ValueError("OCR validation max attempts must be greater than 0.")
+
llm_prompt = prompt_engine.load_prompt("extract-text-from-image")
default_handler = await self._resolve_file_parser_handler(
"extract-text-from-image", workflow_run_block_id, organization_id
@@ -6100,17 +6393,53 @@ class FileParserBlock(Block):
async def _ocr_page(page_image: bytes) -> str:
async with semaphore:
- # OCR transcription intentionally skips system_prompt; it still applies
- # to the downstream extract-information-from-file-text call.
- llm_response = await llm_api_handler(
- prompt=llm_prompt,
- prompt_name="extract-text-from-image",
- screenshots=[page_image],
- force_dict=True,
- workflow_run_block_id=workflow_run_block_id,
- organization_id=organization_id,
- )
- return llm_response.get("extracted_text", "") or ""
+ prompt_for_attempt = llm_prompt
+ for attempt in range(self.ocr_validation_max_attempts):
+ try:
+ # OCR transcription intentionally skips system_prompt; it still applies
+ # to the downstream extract-information-from-file-text call.
+ llm_response = await llm_api_handler(
+ prompt=prompt_for_attempt,
+ prompt_name="extract-text-from-image",
+ screenshots=[page_image],
+ # Schema validation must inspect the raw parsed root; dict coercion can hide bad OCR JSON.
+ force_dict=False,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
+ except (InvalidLLMResponseFormat, InvalidLLMResponseType) as e:
+ failure_reason = _llm_response_format_failure_reason(e)
+ will_retry = attempt + 1 < self.ocr_validation_max_attempts
+ LOG.warning(
+ "FileParserBlock PDF OCR LLM response failed response-format validation",
+ file_url=self.file_url,
+ attempt=attempt + 1,
+ max_attempts=self.ocr_validation_max_attempts,
+ will_retry=will_retry,
+ error_type=type(e).__name__,
+ )
+ if not will_retry:
+ raise ValueError(failure_reason) from e
+ prompt_for_attempt = self._build_ocr_validation_retry_prompt(llm_prompt, failure_reason)
+ continue
+
+ ocr_failure_reason = self._validate_ocr_llm_response(llm_response)
+ if not ocr_failure_reason:
+ return llm_response.get("extracted_text", "") or ""
+
+ will_retry = attempt + 1 < self.ocr_validation_max_attempts
+ LOG.warning(
+ "FileParserBlock PDF OCR LLM response failed schema validation",
+ file_url=self.file_url,
+ attempt=attempt + 1,
+ max_attempts=self.ocr_validation_max_attempts,
+ will_retry=will_retry,
+ failure_reason=ocr_failure_reason,
+ )
+ if not will_retry:
+ raise ValueError(ocr_failure_reason)
+ prompt_for_attempt = self._build_ocr_validation_retry_prompt(llm_prompt, ocr_failure_reason)
+ raise RuntimeError("OCR retry loop exhausted without returning or raising.")
page_results = await asyncio.gather(
*(_ocr_page(page_image) for page_image in page_images),
@@ -6160,6 +6489,9 @@ class FileParserBlock(Block):
organization_id: str | None = None,
) -> str:
"""Parse image file using vision LLM for OCR."""
+ if self.ocr_validation_max_attempts <= 0:
+ raise ValueError("OCR validation max attempts must be greater than 0.")
+
try:
with open(file_path, "rb") as f:
image_bytes = f.read()
@@ -6173,15 +6505,52 @@ class FileParserBlock(Block):
)
# OCR transcription intentionally skips system_prompt — see
# _parse_pdf_file_with_vision_ocr for rationale.
- llm_response = await llm_api_handler(
- prompt=llm_prompt,
- prompt_name="extract-text-from-image",
- screenshots=[image_bytes],
- force_dict=True,
- workflow_run_block_id=workflow_run_block_id,
- organization_id=organization_id,
- )
- return llm_response.get("extracted_text", "")
+ prompt_for_attempt = llm_prompt
+ for attempt in range(self.ocr_validation_max_attempts):
+ try:
+ llm_response = await llm_api_handler(
+ prompt=prompt_for_attempt,
+ prompt_name="extract-text-from-image",
+ screenshots=[image_bytes],
+ # Schema validation must inspect the raw parsed root; dict coercion can hide bad OCR JSON.
+ force_dict=False,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
+ except (InvalidLLMResponseFormat, InvalidLLMResponseType) as e:
+ failure_reason = _llm_response_format_failure_reason(e)
+ will_retry = attempt + 1 < self.ocr_validation_max_attempts
+ LOG.warning(
+ "FileParserBlock image OCR LLM response failed response-format validation",
+ file_url=self.file_url,
+ attempt=attempt + 1,
+ max_attempts=self.ocr_validation_max_attempts,
+ will_retry=will_retry,
+ error_type=type(e).__name__,
+ )
+ if not will_retry:
+ raise ValueError(failure_reason) from e
+ prompt_for_attempt = self._build_ocr_validation_retry_prompt(llm_prompt, failure_reason)
+ continue
+
+ ocr_failure_reason = self._validate_ocr_llm_response(llm_response)
+ if not ocr_failure_reason:
+ return llm_response.get("extracted_text", "") or ""
+
+ will_retry = attempt + 1 < self.ocr_validation_max_attempts
+ LOG.warning(
+ "FileParserBlock image OCR LLM response failed schema validation",
+ file_url=self.file_url,
+ attempt=attempt + 1,
+ max_attempts=self.ocr_validation_max_attempts,
+ will_retry=will_retry,
+ failure_reason=ocr_failure_reason,
+ )
+ if not will_retry:
+ raise ValueError(ocr_failure_reason)
+ prompt_for_attempt = self._build_ocr_validation_retry_prompt(llm_prompt, ocr_failure_reason)
+
+ raise RuntimeError("OCR retry loop exhausted without returning or raising.")
except Exception:
LOG.exception("Failed to extract text from image via OCR", file_url=self.file_url)
raise
@@ -6262,18 +6631,12 @@ class FileParserBlock(Block):
workflow_run_context: WorkflowRunContext,
workflow_run_block_id: str | None = None,
organization_id: str | None = None,
- ) -> dict[str, Any]:
+ ) -> dict[str, Any] | list | str | None:
"""Extract structured data using AI based on json_schema."""
# Use local variable to avoid mutating the instance
- schema_to_use = self.json_schema or {
- "type": "object",
- "properties": {
- "output": {
- "type": "object",
- "description": "Information extracted from the file",
- }
- },
- }
+ schema_to_use = self.json_schema or _default_structured_output_schema("Information extracted from the file")
+ if not validate_schema(schema_to_use):
+ raise ValueError("File parser JSON schema is invalid.")
# Convert content to string for AI processing
if isinstance(content, list):
@@ -6291,15 +6654,58 @@ class FileParserBlock(Block):
)
llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(llm_key, default=default_handler)
- llm_response = await llm_api_handler(
- prompt=llm_prompt,
- prompt_name="extract-information-from-file-text",
- force_dict=False,
- system_prompt=self.workflow_system_prompt,
- workflow_run_block_id=workflow_run_block_id,
- organization_id=organization_id,
- )
- return llm_response
+ prompt_for_attempt = llm_prompt
+ for attempt in range(self.schema_validation_max_attempts):
+ try:
+ llm_response = await llm_api_handler(
+ prompt=prompt_for_attempt,
+ prompt_name="extract-information-from-file-text",
+ # Schema validation must inspect the raw parsed root; dict coercion can hide wrong-root responses.
+ force_dict=False,
+ system_prompt=self.workflow_system_prompt,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
+ except (InvalidLLMResponseFormat, InvalidLLMResponseType) as e:
+ failure_reason = _llm_response_format_failure_reason(e)
+ will_retry = attempt + 1 < self.schema_validation_max_attempts
+ LOG.warning(
+ "FileParserBlock extraction LLM response failed response-format validation",
+ file_url=self.file_url,
+ attempt=attempt + 1,
+ max_attempts=self.schema_validation_max_attempts,
+ will_retry=will_retry,
+ error_type=type(e).__name__,
+ schema_type=schema_to_use.get("type"),
+ )
+ if not will_retry:
+ raise ValueError(failure_reason) from e
+ prompt_for_attempt = _build_schema_validation_retry_prompt(llm_prompt, failure_reason)
+ continue
+
+ schema_validation_failure = self._validate_ai_response_against_json_schema(llm_response, schema_to_use)
+ if not schema_validation_failure:
+ return llm_response
+
+ is_schema_configuration_failure = _is_schema_configuration_failure(schema_validation_failure)
+ will_retry = attempt + 1 < self.schema_validation_max_attempts and not is_schema_configuration_failure
+ LOG.warning(
+ "FileParserBlock extraction LLM response failed schema validation",
+ file_url=self.file_url,
+ attempt=attempt + 1,
+ max_attempts=self.schema_validation_max_attempts,
+ will_retry=will_retry,
+ failure_reason=schema_validation_failure,
+ schema_type=schema_to_use.get("type"),
+ )
+ if not will_retry:
+ raise ValueError(schema_validation_failure)
+ prompt_for_attempt = _build_schema_validation_retry_prompt(
+ llm_prompt,
+ schema_validation_failure,
+ )
+
+ raise AssertionError("unreachable schema validation retry loop exit")
async def _record_failure(
self,
@@ -6468,7 +6874,7 @@ class FileParserBlock(Block):
)
# If json_schema is provided, use AI to extract structured data
- final_data: str | list[dict[str, Any]] | dict[str, Any]
+ final_data: Any
LOG.debug(
"FileParserBlock JSON schema check",
has_json_schema=self.json_schema is not None,
@@ -6521,6 +6927,7 @@ class PDFParserBlock(Block):
file_url: str
json_schema: dict[str, Any] | None = None
+ schema_validation_max_attempts: ClassVar[int] = SCHEMA_VALIDATION_MAX_ATTEMPTS
def get_all_parameters(
self,
@@ -6589,27 +6996,92 @@ class PDFParserBlock(Block):
)
if not self.json_schema:
- self.json_schema = {
- "type": "object",
- "properties": {
- "output": {
- "type": "object",
- "description": "Information extracted from the text",
- }
- },
- }
+ self.json_schema = _default_structured_output_schema("Information extracted from the text")
+ schema_to_use = self.json_schema
+ assert schema_to_use is not None
+ if not validate_schema(schema_to_use):
+ return await self.build_block_result(
+ success=False,
+ failure_reason="File parser JSON schema is invalid.",
+ output_parameter_value=None,
+ status=BlockStatus.failed,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
llm_prompt = prompt_engine.load_prompt(
- "extract-information-from-file-text", extracted_text_content=extracted_text, json_schema=self.json_schema
- )
- llm_response = await app.LLM_API_HANDLER(
- prompt=llm_prompt,
- prompt_name="extract-information-from-file-text",
- force_dict=False,
- system_prompt=self.workflow_system_prompt,
- workflow_run_block_id=workflow_run_block_id,
- organization_id=organization_id,
+ "extract-information-from-file-text", extracted_text_content=extracted_text, json_schema=schema_to_use
)
+
+ llm_response: dict[str, Any] | list | str | None = None
+ prompt_for_attempt = llm_prompt
+ for attempt in range(self.schema_validation_max_attempts):
+ try:
+ llm_response = await app.LLM_API_HANDLER(
+ prompt=prompt_for_attempt,
+ prompt_name="extract-information-from-file-text",
+ # Schema validation must inspect the raw parsed root; dict coercion can hide wrong-root responses.
+ force_dict=False,
+ system_prompt=self.workflow_system_prompt,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
+ except (InvalidLLMResponseFormat, InvalidLLMResponseType) as e:
+ failure_reason = _llm_response_format_failure_reason(e)
+ will_retry = attempt + 1 < self.schema_validation_max_attempts
+ LOG.warning(
+ "PDFParserBlock extraction LLM response failed response-format validation",
+ file_url=self.file_url,
+ attempt=attempt + 1,
+ max_attempts=self.schema_validation_max_attempts,
+ will_retry=will_retry,
+ error_type=type(e).__name__,
+ schema_type=schema_to_use.get("type"),
+ )
+ if not will_retry:
+ return await self.build_block_result(
+ success=False,
+ failure_reason=failure_reason,
+ output_parameter_value=None,
+ status=BlockStatus.failed,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
+ prompt_for_attempt = _build_schema_validation_retry_prompt(llm_prompt, failure_reason)
+ continue
+
+ schema_validation_failure = FileParserBlock._validate_ai_response_against_json_schema(
+ llm_response,
+ schema_to_use,
+ )
+ if not schema_validation_failure:
+ break
+
+ is_schema_configuration_failure = _is_schema_configuration_failure(schema_validation_failure)
+ will_retry = attempt + 1 < self.schema_validation_max_attempts and not is_schema_configuration_failure
+ LOG.warning(
+ "PDFParserBlock extraction LLM response failed schema validation",
+ file_url=self.file_url,
+ attempt=attempt + 1,
+ max_attempts=self.schema_validation_max_attempts,
+ will_retry=will_retry,
+ failure_reason=schema_validation_failure,
+ schema_type=schema_to_use.get("type"),
+ )
+ if not will_retry:
+ return await self.build_block_result(
+ success=False,
+ failure_reason=schema_validation_failure,
+ output_parameter_value=None,
+ status=BlockStatus.failed,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
+ prompt_for_attempt = _build_schema_validation_retry_prompt(
+ llm_prompt,
+ schema_validation_failure,
+ )
+
# Record the parsed data
await self.record_output_parameter_value(workflow_run_context, workflow_run_id, llm_response)
return await self.build_block_result(
diff --git a/tests/unit/test_text_prompt_block.py b/tests/unit/test_text_prompt_block.py
index 933ba9d4f..6b65203c1 100644
--- a/tests/unit/test_text_prompt_block.py
+++ b/tests/unit/test_text_prompt_block.py
@@ -9,11 +9,12 @@ from jinja2.sandbox import SandboxedEnvironment
import skyvern.forge.sdk.workflow.models.block as _block_mod
from skyvern.config import settings as base_settings
from skyvern.forge.prompts import prompt_engine
+from skyvern.forge.sdk.api.llm.exceptions import InvalidLLMResponseFormat
from skyvern.forge.sdk.settings_manager import SettingsManager
from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
from skyvern.forge.sdk.workflow.models.block import TextPromptBlock
from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType
-from skyvern.schemas.workflows import TextPromptBlockYAML, WorkflowRequest
+from skyvern.schemas.workflows import BlockStatus, TextPromptBlockYAML, WorkflowRequest
block_module = sys.modules["skyvern.forge.sdk.workflow.models.block"]
@@ -128,6 +129,7 @@ async def test_text_prompt_block_uses_selected_model(monkeypatch, model_name):
assert captured["llm_key"] == expected_llm_key
assert captured["prompt_name"] == "text-prompt"
assert response == {"llm_response": "ok"}
+ assert block.json_schema is None
@pytest.mark.asyncio
@@ -401,6 +403,434 @@ async def test_text_prompt_block_uses_explicit_internal_llm_key_override(monkeyp
assert response == {"llm_response": "override"}
+async def test_text_prompt_block_array_schema_does_not_force_dict(monkeypatch):
+ block = _make_text_prompt_block(
+ json_schema={
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "first_name": {"type": "string"},
+ "last_name": {"type": "string"},
+ },
+ "required": ["first_name", "last_name"],
+ },
+ },
+ )
+
+ captured: dict[str, object] = {}
+
+ async def fake_handler(*, prompt: str, prompt_name: str, force_dict: bool, **kwargs):
+ captured["prompt"] = prompt
+ captured["prompt_name"] = prompt_name
+ captured["force_dict"] = force_dict
+ return [{"first_name": "Zachary", "last_name": "Leibrand"}]
+
+ async def fake_resolve_default_llm_handler(*args, **kwargs):
+ return fake_handler
+
+ def fake_get_override_handler(llm_key: str | None, *, default):
+ return default
+
+ monkeypatch.setattr(
+ block_module.LLMAPIHandlerFactory,
+ "get_override_llm_api_handler",
+ fake_get_override_handler,
+ raising=False,
+ )
+ monkeypatch.setattr(
+ TextPromptBlock,
+ "_resolve_default_llm_handler",
+ fake_resolve_default_llm_handler,
+ raising=False,
+ )
+ monkeypatch.setattr(
+ prompt_engine,
+ "load_prompt_from_string",
+ lambda template, **kwargs: template,
+ )
+
+ response = await block.send_prompt(block.prompt, {}, workflow_run_id="workflow-run", organization_id="org-1")
+
+ assert captured["prompt_name"] == "text-prompt"
+ assert captured["force_dict"] is False
+ assert response == [{"first_name": "Zachary", "last_name": "Leibrand"}]
+
+
+async def test_text_prompt_block_object_schema_does_not_force_dict_before_validation(monkeypatch):
+ block = _make_text_prompt_block(
+ json_schema={
+ "type": "object",
+ "properties": {
+ "invoice_search_string": {"type": "string"},
+ },
+ "required": ["invoice_search_string"],
+ },
+ )
+
+ captured: dict[str, object] = {}
+
+ async def fake_handler(*, prompt: str, prompt_name: str, force_dict: bool, **kwargs):
+ captured["force_dict"] = force_dict
+ return {"invoice_search_string": "062026"}
+
+ async def fake_resolve_default_llm_handler(*args, **kwargs):
+ return fake_handler
+
+ def fake_get_override_handler(llm_key: str | None, *, default):
+ return default
+
+ monkeypatch.setattr(
+ block_module.LLMAPIHandlerFactory,
+ "get_override_llm_api_handler",
+ fake_get_override_handler,
+ raising=False,
+ )
+ monkeypatch.setattr(
+ TextPromptBlock,
+ "_resolve_default_llm_handler",
+ fake_resolve_default_llm_handler,
+ raising=False,
+ )
+ monkeypatch.setattr(
+ prompt_engine,
+ "load_prompt_from_string",
+ lambda template, **kwargs: template,
+ )
+
+ response = await block.send_prompt(block.prompt, {}, workflow_run_id="workflow-run", organization_id="org-1")
+
+ assert captured["force_dict"] is False
+ assert response == {"invoice_search_string": "062026"}
+
+
+async def test_text_prompt_block_retry_feedback_is_not_rendered_as_template(monkeypatch):
+ block = _make_text_prompt_block(
+ prompt="Hello {{ name }}",
+ json_schema={
+ "type": "object",
+ "properties": {"answer": {"type": "string"}},
+ "required": ["answer"],
+ },
+ )
+ captured: dict[str, object] = {}
+ rendered_templates: list[str] = []
+
+ async def fake_handler(*, prompt: str, prompt_name: str, force_dict: bool, **kwargs):
+ captured["prompt"] = prompt
+ return {"answer": "ok"}
+
+ async def fake_resolve_default_llm_handler(*args, **kwargs):
+ return fake_handler
+
+ def fake_get_override_handler(llm_key: str | None, *, default):
+ return default
+
+ def fake_load_prompt_from_string(template: str, **kwargs: str) -> str:
+ rendered_templates.append(template)
+ assert "previous response failed JSON schema validation" not in template
+ return template.replace("{{ name }}", kwargs["name"])
+
+ monkeypatch.setattr(
+ block_module.LLMAPIHandlerFactory,
+ "get_override_llm_api_handler",
+ fake_get_override_handler,
+ raising=False,
+ )
+ monkeypatch.setattr(
+ TextPromptBlock,
+ "_resolve_default_llm_handler",
+ fake_resolve_default_llm_handler,
+ raising=False,
+ )
+ monkeypatch.setattr(prompt_engine, "load_prompt_from_string", fake_load_prompt_from_string)
+
+ await block.send_prompt(
+ block.prompt,
+ {"name": "Alice"},
+ workflow_run_id="workflow-run",
+ organization_id="org-1",
+ schema_validation_failure="root: has 1 unexpected properties {{ dangerous_lookup }}",
+ )
+
+ sent_prompt = captured["prompt"]
+ assert isinstance(sent_prompt, str)
+ assert "Hello Alice" in sent_prompt
+ assert "previous response failed JSON schema validation" in sent_prompt
+ assert "{{ dangerous_lookup }}" in sent_prompt
+
+
+def test_text_prompt_block_schema_validation_rejects_wrong_root_type() -> None:
+ block = _make_text_prompt_block(
+ json_schema={
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "first_name": {"type": "string"},
+ "last_name": {"type": "string"},
+ },
+ "required": ["first_name", "last_name"],
+ },
+ },
+ )
+
+ failure_reason = block._validate_response_against_json_schema(
+ {"first_name": "f_name_var", "last_name": "l_name_var\n```python\n..."}
+ )
+
+ assert failure_reason is not None
+ assert "does not match text prompt JSON schema" in failure_reason
+ assert "root" in failure_reason
+ assert "expected type array, got object" in failure_reason
+
+
+def test_text_prompt_block_schema_validation_feedback_does_not_echo_invalid_response() -> None:
+ secret_like_value = "customer-private-value-" + ("x" * 300)
+ block = _make_text_prompt_block(
+ json_schema={
+ "type": "object",
+ "properties": {
+ "values": {
+ "type": "array",
+ "items": {"type": "string"},
+ },
+ },
+ "required": ["values"],
+ },
+ )
+
+ failure_reason = block._validate_response_against_json_schema({"values": secret_like_value})
+
+ assert failure_reason is not None
+ assert "root.values: expected type array, got string" in failure_reason
+ assert secret_like_value not in failure_reason
+ assert "customer-private-value" not in failure_reason
+
+
+def test_text_prompt_block_additional_properties_feedback_does_not_echo_response_keys() -> None:
+ block = _make_text_prompt_block(
+ json_schema={
+ "type": "object",
+ "properties": {"safe": {"type": "string"}},
+ "additionalProperties": False,
+ },
+ )
+
+ failure_reason = block._validate_response_against_json_schema({"safe": "ok", "[email protected]": "private"})
+
+ assert failure_reason is not None
+ assert "has 1 unexpected properties" in failure_reason
+ assert "[email protected]" not in failure_reason
+ assert "private" not in failure_reason
+
+
+def test_text_prompt_block_map_value_path_does_not_echo_response_key() -> None:
+ block = _make_text_prompt_block(
+ json_schema={
+ "type": "object",
+ "additionalProperties": {"type": "number"},
+ },
+ )
+
+ failure_reason = block._validate_response_against_json_schema({"[email protected]": "not-a-number"})
+
+ assert failure_reason is not None
+ assert "root.<map value>: expected type number, got string" in failure_reason
+ assert "[email protected]" not in failure_reason
+ assert "not-a-number" not in failure_reason
+
+
+def test_text_prompt_block_unresolvable_ref_fails_schema_validation_gracefully() -> None:
+ block = _make_text_prompt_block(json_schema={"$ref": "#/$defs/missing"})
+
+ failure_reason = block._validate_response_against_json_schema({"value": "ok"})
+
+ assert failure_reason is not None
+ assert "Text prompt JSON schema validation failed" in failure_reason
+
+
+async def test_text_prompt_block_execute_fails_schema_validation_before_recording(monkeypatch):
+ block = _make_text_prompt_block(
+ json_schema={
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "first_name": {"type": "string"},
+ "last_name": {"type": "string"},
+ },
+ "required": ["first_name", "last_name"],
+ },
+ },
+ )
+ workflow_run_context = _make_workflow_run_context()
+ bad_response = {"first_name": "f_name_var", "last_name": "code fragment"}
+ record_output_parameter_value = AsyncMock()
+ send_prompt = AsyncMock(return_value=bad_response)
+
+ monkeypatch.setattr(TextPromptBlock, "send_prompt", send_prompt)
+ monkeypatch.setattr(
+ TextPromptBlock,
+ "get_workflow_run_context",
+ staticmethod(lambda workflow_run_id: workflow_run_context),
+ )
+ monkeypatch.setattr(TextPromptBlock, "record_output_parameter_value", record_output_parameter_value)
+
+ result = await block.execute(
+ workflow_run_id="workflow-run",
+ workflow_run_block_id="workflow-run-block",
+ organization_id="org-1",
+ )
+
+ assert result.success is False
+ assert result.status == BlockStatus.failed
+ assert result.output_parameter_value is None
+ assert result.failure_reason is not None
+ assert "does not match text prompt JSON schema" in result.failure_reason
+ record_output_parameter_value.assert_not_awaited()
+ assert send_prompt.await_count == block.schema_validation_max_attempts
+
+
+async def test_text_prompt_block_execute_invalid_schema_fails_without_llm_retry(monkeypatch):
+ block = _make_text_prompt_block(json_schema={"type": 123})
+ workflow_run_context = _make_workflow_run_context()
+ record_output_parameter_value = AsyncMock()
+ send_prompt = AsyncMock(return_value={"llm_response": "ok"})
+
+ monkeypatch.setattr(TextPromptBlock, "send_prompt", send_prompt)
+ monkeypatch.setattr(
+ TextPromptBlock,
+ "get_workflow_run_context",
+ staticmethod(lambda workflow_run_id: workflow_run_context),
+ )
+ monkeypatch.setattr(TextPromptBlock, "record_output_parameter_value", record_output_parameter_value)
+
+ result = await block.execute(
+ workflow_run_id="workflow-run",
+ workflow_run_block_id="workflow-run-block",
+ organization_id="org-1",
+ )
+
+ assert result.success is False
+ assert result.status == BlockStatus.failed
+ assert result.failure_reason == "Text prompt JSON schema is invalid."
+ send_prompt.assert_not_awaited()
+ record_output_parameter_value.assert_not_awaited()
+
+
+async def test_text_prompt_block_execute_retries_schema_validation_failure(monkeypatch):
+ block = _make_text_prompt_block(
+ json_schema={
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "first_name": {"type": "string"},
+ "last_name": {"type": "string"},
+ },
+ "required": ["first_name", "last_name"],
+ },
+ },
+ )
+ workflow_run_context = _make_workflow_run_context()
+ bad_response = {"first_name": "f_name_var", "last_name": "code fragment"}
+ good_response = [{"first_name": "Zachary", "last_name": "Leibrand"}]
+ responses = [bad_response, good_response]
+ prompts: list[str] = []
+ retry_failures: list[str | None] = []
+ record_output_parameter_value = AsyncMock()
+
+ async def fake_send_prompt(self, prompt, *args, **kwargs):
+ prompts.append(prompt)
+ retry_failures.append(kwargs.get("schema_validation_failure"))
+ return responses.pop(0)
+
+ monkeypatch.setattr(TextPromptBlock, "send_prompt", fake_send_prompt)
+ monkeypatch.setattr(
+ TextPromptBlock,
+ "get_workflow_run_context",
+ staticmethod(lambda workflow_run_id: workflow_run_context),
+ )
+ monkeypatch.setattr(TextPromptBlock, "record_output_parameter_value", record_output_parameter_value)
+
+ result = await block.execute(
+ workflow_run_id="workflow-run",
+ workflow_run_block_id="workflow-run-block",
+ organization_id="org-1",
+ )
+
+ assert result.success is True
+ assert result.status == BlockStatus.completed
+ assert result.output_parameter_value == good_response
+ assert prompts == [block.prompt, block.prompt]
+ assert retry_failures[0] is None
+ assert retry_failures[1] is not None
+ assert "expected type array, got object" in retry_failures[1]
+ record_output_parameter_value.assert_awaited_once_with(workflow_run_context, "workflow-run", good_response)
+
+
+async def test_text_prompt_block_execute_retries_response_format_failure(monkeypatch):
+ block = _make_text_prompt_block(
+ json_schema={
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "first_name": {"type": "string"},
+ "last_name": {"type": "string"},
+ },
+ "required": ["first_name", "last_name"],
+ },
+ },
+ )
+ workflow_run_context = _make_workflow_run_context()
+ raw_bad_response = "not-json-customer-private-value-" + ("x" * 300)
+ good_response = [{"first_name": "Zachary", "last_name": "Leibrand"}]
+ prompts: list[str] = []
+ retry_failures: list[str | None] = []
+ record_output_parameter_value = AsyncMock()
+
+ async def fake_send_prompt(self, prompt, *args, **kwargs):
+ prompts.append(prompt)
+ retry_failures.append(kwargs.get("schema_validation_failure"))
+ if len(prompts) == 1:
+ raise InvalidLLMResponseFormat(raw_bad_response)
+ return good_response
+
+ monkeypatch.setattr(TextPromptBlock, "send_prompt", fake_send_prompt)
+ monkeypatch.setattr(
+ TextPromptBlock,
+ "get_workflow_run_context",
+ staticmethod(lambda workflow_run_id: workflow_run_context),
+ )
+ monkeypatch.setattr(TextPromptBlock, "record_output_parameter_value", record_output_parameter_value)
+
+ result = await block.execute(
+ workflow_run_id="workflow-run",
+ workflow_run_block_id="workflow-run-block",
+ organization_id="org-1",
+ )
+
+ assert result.success is True
+ assert result.status == BlockStatus.completed
+ assert result.output_parameter_value == good_response
+ assert prompts == [block.prompt, block.prompt]
+ assert retry_failures[0] is None
+ assert retry_failures[1] is not None
+ assert "InvalidLLMResponseFormat" in retry_failures[1]
+ assert raw_bad_response not in retry_failures[1]
+ assert "customer-private-value" not in retry_failures[1]
+ record_output_parameter_value.assert_awaited_once_with(workflow_run_context, "workflow-run", good_response)
+
+
def test_workflow_request_deserialization_strips_invalid_text_prompt_llm_key() -> None:
"""Verify FastAPI deserialization (not just explicit model_validate) strips bad keys.
diff --git a/tests/unit/workflow/test_file_parser_block.py b/tests/unit/workflow/test_file_parser_block.py
index ee36a7dad..23f64a9c4 100644
--- a/tests/unit/workflow/test_file_parser_block.py
+++ b/tests/unit/workflow/test_file_parser_block.py
@@ -17,10 +17,11 @@ from unittest.mock import AsyncMock, MagicMock
import docx
import pytest
+from skyvern.forge.sdk.api.llm.exceptions import InvalidLLMResponseFormat
from skyvern.forge.sdk.workflow.exceptions import InvalidFileType
-from skyvern.forge.sdk.workflow.models.block import BlockType, FileParserBlock
+from skyvern.forge.sdk.workflow.models.block import BlockType, FileParserBlock, PDFParserBlock
from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType
-from skyvern.schemas.workflows import FileType
+from skyvern.schemas.workflows import BlockResult, BlockStatus, FileType
def _make_output_parameter(key: str) -> OutputParameter:
@@ -45,6 +46,15 @@ def _make_file_parser_block(file_url: str, file_type: FileType) -> FileParserBlo
)
+def _make_pdf_parser_block(file_url: str) -> PDFParserBlock:
+ return PDFParserBlock(
+ label="test_pdf_parser",
+ block_type=BlockType.PDF_PARSER,
+ output_parameter=_make_output_parameter("test_output"),
+ file_url=file_url,
+ )
+
+
def _create_docx(
path: Path,
paragraphs: list[str] | None = None,
@@ -384,6 +394,208 @@ class TestExtractWithAiSerialization:
assert kwargs["extracted_text_content"] == "Hello\nWorld"
+class TestExtractWithAiSchemaValidation:
+ """Tests for schema adherence in FileParserBlock AI extraction."""
+
+ @staticmethod
+ def _patch_prompt_and_handler(mp: pytest.MonkeyPatch, handler: AsyncMock, prompt: str = "base prompt") -> None:
+ mp.setattr(
+ "skyvern.forge.sdk.workflow.models.block.LLMAPIHandlerFactory.get_override_llm_api_handler",
+ lambda *a, **kw: handler,
+ )
+ mp.setattr(
+ "skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt",
+ MagicMock(return_value=prompt),
+ )
+
+ async def test_array_schema_does_not_force_dict(self) -> None:
+ block = _make_file_parser_block("https://example.com/data.csv", FileType.CSV)
+ block.json_schema = {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {"name": {"type": "string"}},
+ "required": ["name"],
+ },
+ }
+ captured: dict[str, object] = {}
+
+ async def fake_handler(**kwargs: Any) -> list[dict[str, str]]:
+ captured["force_dict"] = kwargs["force_dict"]
+ return [{"name": "Alice"}]
+
+ with pytest.MonkeyPatch.context() as mp:
+ handler = AsyncMock(side_effect=fake_handler)
+ self._patch_prompt_and_handler(mp, handler)
+
+ result = await block._extract_with_ai("name\nAlice", MagicMock())
+
+ assert captured["force_dict"] is False
+ assert result == [{"name": "Alice"}]
+
+ async def test_object_schema_wrong_root_retries_without_prevalidation_coercion(self) -> None:
+ block = _make_file_parser_block("https://example.com/data.csv", FileType.CSV)
+ block.json_schema = {
+ "type": "object",
+ "properties": {"name": {"type": "string"}},
+ "required": ["name"],
+ }
+ responses: list[Any] = [[{"name": "Alice"}], {"name": "Alice"}]
+ prompts: list[str] = []
+ force_dict_values: list[bool] = []
+
+ async def fake_handler(**kwargs: Any) -> Any:
+ prompts.append(kwargs["prompt"])
+ force_dict_values.append(kwargs["force_dict"])
+ return responses.pop(0)
+
+ with pytest.MonkeyPatch.context() as mp:
+ handler = AsyncMock(side_effect=fake_handler)
+ self._patch_prompt_and_handler(mp, handler)
+
+ result = await block._extract_with_ai("name\nAlice", MagicMock())
+
+ assert result == {"name": "Alice"}
+ assert handler.await_count == 2
+ assert force_dict_values == [False, False]
+ assert "previous response failed JSON schema validation" in prompts[1]
+ assert "expected type object, got array" in prompts[1]
+
+ async def test_invalid_schema_fails_before_llm_retry(self) -> None:
+ block = _make_file_parser_block("https://example.com/data.csv", FileType.CSV)
+ block.json_schema = {"type": 123}
+
+ with pytest.MonkeyPatch.context() as mp:
+ handler = AsyncMock()
+ self._patch_prompt_and_handler(mp, handler)
+
+ with pytest.raises(ValueError, match="File parser JSON schema is invalid"):
+ await block._extract_with_ai("name\nAlice", MagicMock())
+
+ handler.assert_not_awaited()
+
+ async def test_schema_validation_failure_retries_with_sanitized_prompt(self) -> None:
+ block = _make_file_parser_block("https://example.com/data.csv", FileType.CSV)
+ block.json_schema = {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {"name": {"type": "string"}},
+ "required": ["name"],
+ },
+ }
+ secret_like_value = "customer-private-value-" + ("x" * 300)
+ responses: list[Any] = [{"name": secret_like_value}, [{"name": "Alice"}]]
+ prompts: list[str] = []
+
+ async def fake_handler(**kwargs: Any) -> Any:
+ prompts.append(kwargs["prompt"])
+ return responses.pop(0)
+
+ with pytest.MonkeyPatch.context() as mp:
+ handler = AsyncMock(side_effect=fake_handler)
+ self._patch_prompt_and_handler(mp, handler)
+
+ result = await block._extract_with_ai("name\nAlice", MagicMock())
+
+ assert result == [{"name": "Alice"}]
+ assert handler.await_count == 2
+ assert prompts[0] == "base prompt"
+ assert "previous response failed JSON schema validation" in prompts[1]
+ assert "expected type array, got object" in prompts[1]
+ assert secret_like_value not in prompts[1]
+ assert "customer-private-value" not in prompts[1]
+
+ async def test_response_format_failure_retries_with_sanitized_prompt(self) -> None:
+ block = _make_file_parser_block("https://example.com/data.csv", FileType.CSV)
+ block.json_schema = {
+ "type": "object",
+ "properties": {"name": {"type": "string"}},
+ "required": ["name"],
+ }
+ raw_bad_response = "not-json-customer-private-value-" + ("x" * 300)
+ prompts: list[str] = []
+
+ async def fake_handler(**kwargs: Any) -> dict[str, str]:
+ prompts.append(kwargs["prompt"])
+ if len(prompts) == 1:
+ raise InvalidLLMResponseFormat(raw_bad_response)
+ return {"name": "Alice"}
+
+ with pytest.MonkeyPatch.context() as mp:
+ handler = AsyncMock(side_effect=fake_handler)
+ self._patch_prompt_and_handler(mp, handler)
+
+ result = await block._extract_with_ai("name\nAlice", MagicMock())
+
+ assert result == {"name": "Alice"}
+ assert handler.await_count == 2
+ assert "InvalidLLMResponseFormat" in prompts[1]
+ assert raw_bad_response not in prompts[1]
+ assert "customer-private-value" not in prompts[1]
+
+
+class TestPDFParserSchemaValidation:
+ """Tests for schema adherence in the deprecated PDFParserBlock."""
+
+ @staticmethod
+ def _patch_execute_dependencies(mp: pytest.MonkeyPatch, block: PDFParserBlock, handler: AsyncMock) -> AsyncMock:
+ workflow_run_context = MagicMock()
+ workflow_run_context.has_parameter.return_value = False
+ record_output_parameter_value = AsyncMock()
+
+ async def fake_build_block_result(self: PDFParserBlock, **kwargs: Any) -> BlockResult:
+ kwargs.pop("organization_id", None)
+ return BlockResult(output_parameter=self.output_parameter, **kwargs)
+
+ mp.setattr(
+ PDFParserBlock, "get_workflow_run_context", staticmethod(lambda workflow_run_id: workflow_run_context)
+ )
+ mp.setattr(PDFParserBlock, "record_output_parameter_value", record_output_parameter_value)
+ mp.setattr(PDFParserBlock, "build_block_result", fake_build_block_result)
+ mp.setattr("skyvern.forge.sdk.workflow.models.block.download_file", AsyncMock(return_value="/tmp/test.pdf"))
+ mp.setattr("skyvern.forge.sdk.workflow.models.block.extract_pdf_file", MagicMock(return_value="name\nAlice"))
+ mp.setattr(
+ "skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt",
+ MagicMock(return_value="base prompt"),
+ )
+ mp.setattr("skyvern.forge.sdk.workflow.models.block.app.LLM_API_HANDLER", handler)
+ return record_output_parameter_value
+
+ async def test_execute_retries_schema_validation_without_prevalidation_coercion(self) -> None:
+ block = _make_pdf_parser_block("https://example.com/data.pdf")
+ block.json_schema = {
+ "type": "object",
+ "properties": {"name": {"type": "string"}},
+ "required": ["name"],
+ }
+ responses: list[Any] = [[{"name": "Alice"}], {"name": "Alice"}]
+ prompts: list[str] = []
+ force_dict_values: list[bool] = []
+
+ async def fake_handler(**kwargs: Any) -> Any:
+ prompts.append(kwargs["prompt"])
+ force_dict_values.append(kwargs["force_dict"])
+ return responses.pop(0)
+
+ with pytest.MonkeyPatch.context() as mp:
+ handler = AsyncMock(side_effect=fake_handler)
+ record_output_parameter_value = self._patch_execute_dependencies(mp, block, handler)
+
+ result = await block.execute("workflow-run", "workflow-run-block", organization_id="org-1")
+
+ assert result.success is True
+ assert result.status == BlockStatus.completed
+ assert result.output_parameter_value == {"name": "Alice"}
+ assert handler.await_count == 2
+ assert force_dict_values == [False, False]
+ assert "previous response failed JSON schema validation" in prompts[1]
+ assert "expected type object, got array" in prompts[1]
+ record_output_parameter_value.assert_awaited_once()
+
+
@pytest.mark.asyncio
class TestOcrPdfPages:
"""Tests for the per-page vision-LLM OCR path for scanned PDFs (SKY-10960)."""
@@ -427,6 +639,34 @@ class TestOcrPdfPages:
assert "Cover sheet" in result and "Demographics" in result
assert result.index("--- Page 1 ---") < result.index("--- Page 2 ---") < result.index("--- Page 3 ---")
+ async def test_invalid_ocr_response_retries_with_sanitized_prompt(self) -> None:
+ """A missing extracted_text field is retried without echoing invalid content."""
+ block = _make_file_parser_block("https://example.com/scan.pdf", FileType.PDF)
+ secret_like_value = "customer-private-value-" + ("x" * 300)
+ prompts: list[str] = []
+ force_dict_values: list[bool] = []
+
+ async def fake_handler(**kwargs: Any) -> dict[str, str]:
+ prompts.append(kwargs["prompt"])
+ force_dict_values.append(kwargs["force_dict"])
+ if len(prompts) == 1:
+ return {"wrong_field": secret_like_value}
+ return {"extracted_text": "Recovered page text"}
+
+ with pytest.MonkeyPatch.context() as mp:
+ handler = AsyncMock(side_effect=fake_handler)
+ self._patch_handler(mp, handler)
+
+ result = await block._ocr_pdf_pages([b"img-1"])
+
+ assert handler.await_count == 2
+ assert force_dict_values == [False, False]
+ assert "Recovered page text" in result
+ assert "previous OCR response failed JSON validation" in prompts[1]
+ assert "must include extracted_text as a string" in prompts[1]
+ assert secret_like_value not in prompts[1]
+ assert "customer-private-value" not in prompts[1]
+
async def test_order_preserved_when_a_later_page_resolves_first(self) -> None:
"""Output stays in page order even if an earlier page's call finishes last."""
block = _make_file_parser_block("https://example.com/scan.pdf", FileType.PDF)