Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix: enforce LLM block schema outputs (#6965)
Suchintan4ee1e133656d
diff --git a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsxindex 4dc059c4a..e9b24497b 100644--- a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx@@ -103,6 +103,7 @@ import { DebuggerBlockRuns } from "@/routes/workflows/debugger/DebuggerBlockRunsimport { copyText } from "@/util/copyText";import { isMacPlatform } from "@/util/platform";import { parseHeaderJson } from "@/util/secretHeaders";+import { getJsonParseErrorDetail } from "@/util/jsonParseError";import { cn } from "@/util/utils";import { FlowRenderer, type FlowRendererProps } from "./FlowRenderer";@@ -2432,7 +2433,10 @@ function Workspace({} catch (error) {toast({title: "Error",- description: "Invalid JSON format in extra http headers",+ description: `Invalid JSON format in extra http headers: ${getJsonParseErrorDetail(+ saveData.settings.extraHttpHeaders ?? "",+ error,+ )}`,variant: "destructive",});return;@@ -2448,7 +2452,10 @@ function Workspace({} catch (error) {toast({title: "Error",- description: "Invalid JSON format in cdp connect headers",+ description: `Invalid JSON format in cdp connect headers: ${getJsonParseErrorDetail(+ saveData.settings.cdpConnectHeaders ?? "",+ error,+ )}`,variant: "destructive",});return;diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/FileParserEditor.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/FileParserEditor.tsxindex 2eac9a9c0..8037f452f 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/FileParserEditor.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/FileParserEditor.tsx@@ -2,6 +2,12 @@ import { useReactFlow } from "@xyflow/react";import { HelpTooltip } from "@/components/HelpTooltip";import { ModelSelector } from "@/components/ModelSelector";+import {+ Accordion,+ AccordionContent,+ AccordionItem,+ AccordionTrigger,+} from "@/components/ui/accordion";import { Label } from "@/components/ui/label";import {Select,@@ -153,13 +159,26 @@ function FileParserEditorBody({value={data.model}onChange={(value) => update({ model: value })}/>- <IgnoreWorkflowSystemPrompt- ignoreWorkflowSystemPrompt={data.ignoreWorkflowSystemPrompt ?? false}- editable={editable}- onIgnoreWorkflowSystemPromptChange={(ignoreWorkflowSystemPrompt) => {- update({ ignoreWorkflowSystemPrompt });- }}- />+ <Accordion type="single" collapsible>+ <AccordionItem value="advanced" className="border-b-0">+ <AccordionTrigger className="py-0">+ Advanced Settings+ </AccordionTrigger>+ <AccordionContent className="pl-6 pr-1 pt-1">+ <IgnoreWorkflowSystemPrompt+ ignoreWorkflowSystemPrompt={+ data.ignoreWorkflowSystemPrompt ?? false+ }+ editable={editable}+ onIgnoreWorkflowSystemPromptChange={(+ ignoreWorkflowSystemPrompt,+ ) => {+ update({ ignoreWorkflowSystemPrompt });+ }}+ />+ </AccordionContent>+ </AccordionItem>+ </Accordion></div>);}diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/GoogleSheetsWriteNode/validate.ts b/skyvern-frontend/src/routes/workflows/editor/nodes/GoogleSheetsWriteNode/validate.tsindex b9cac7c83..24d0b55dc 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/GoogleSheetsWriteNode/validate.ts+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/GoogleSheetsWriteNode/validate.ts@@ -1,3 +1,5 @@+import { getJsonParseErrorDetail } from "@/util/jsonParseError";+import type { GoogleSheetsWriteNode } from "./types";const missing = (value: string): boolean => !value.trim();@@ -35,8 +37,13 @@ export function validateGoogleSheetsWriteNode(`${label}: Column mapping must be a JSON object of source field -> column letter.`,);}- } catch {- errors.push(`${label}: Column mapping is not valid JSON.`);+ } catch (error) {+ errors.push(+ `${label}: Column mapping is not valid JSON: ${getJsonParseErrorDetail(+ mapping,+ error,+ )}`,+ );}}return errors;diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/HttpUtils.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/HttpUtils.tsxindex c71345fb1..367619e98 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/HttpUtils.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/HttpUtils.tsx@@ -64,18 +64,18 @@ export function UrlValidator({ url }: { url: string }) {return (<divclassName={cn(- "flex items-center gap-1 text-xs",+ "flex min-w-0 items-start gap-1 text-xs",validation.valid? "text-green-600 dark:text-green-400": "text-red-600 dark:text-red-400",)}>{validation.valid ? (- <CheckCircledIcon className="h-3 w-3" />+ <CheckCircledIcon className="h-3 w-3 shrink-0" />) : (- <ExclamationTriangleIcon className="h-3 w-3" />+ <ExclamationTriangleIcon className="h-3 w-3 shrink-0" />)}- <span>{validation.message}</span>+ <span className="min-w-0 break-words">{validation.message}</span></div>);}@@ -89,18 +89,18 @@ export function JsonValidator({ value }: { value: string }) {return (<divclassName={cn(- "flex items-center gap-1 text-xs",+ "flex min-w-0 items-start gap-1 text-xs",validation.valid? "text-green-600 dark:text-green-400": "text-red-600 dark:text-red-400",)}>{validation.valid ? (- <CheckCircledIcon className="h-3 w-3" />+ <CheckCircledIcon className="h-3 w-3 shrink-0" />) : (- <ExclamationTriangleIcon className="h-3 w-3" />+ <ExclamationTriangleIcon className="h-3 w-3 shrink-0" />)}- <span>{validation.message}</span>+ <span className="min-w-0 break-words">{validation.message}</span></div>);}diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/httpValidation.test.ts b/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/httpValidation.test.tsindex ea6e8521a..ba76a7475 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/httpValidation.test.ts+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/httpValidation.test.ts@@ -61,6 +61,19 @@ describe("validateJson", () => {expect(result.valid).toBe(false);});+ test("non-breaking spaces outside strings explain the hidden character", () => {+ const result = validateJson(`{+ "personalInfo": {+ "firstName": "Luis",+ "lastName": "Ortiz"\u00a0+ }+}`);+ expect(result.valid).toBe(false);+ expect(result.message).toContain(+ "JSON contains a non-breaking space (U+00A0) at line 4, column 24; replace it with a regular space.",+ );+ });+test("completely malformed JSON is invalid", () => {const result = validateJson("{key: value}");expect(result.valid).toBe(false);diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/httpValidation.ts b/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/httpValidation.tsindex 23c9abd46..d1919560b 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/httpValidation.ts+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/httpValidation.ts@@ -1,4 +1,5 @@import { TSON } from "@/util/tson";+import { getInvalidJsonMessage } from "@/util/jsonParseError";// URL Validation Helperexport function validateUrl(url: string): { valid: boolean; message: string } {@@ -31,7 +32,13 @@ export function validateJson(value: string): {// First check: does it pass TSON (template-aware) parsing?const tsonResult = TSON.parse(trimmed);if (!tsonResult.success) {- return { valid: false, message: tsonResult.error || "Invalid JSON" };+ return {+ valid: false,+ message: getInvalidJsonMessage(+ trimmed,+ tsonResult.error || "Parse error",+ ),+ };}// Second check: does it also pass strict JSON parsing?diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/PDFParserNode/PDFParserEditor.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/PDFParserNode/PDFParserEditor.tsxindex 6aabe3deb..803373f05 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/PDFParserNode/PDFParserEditor.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/PDFParserNode/PDFParserEditor.tsx@@ -2,6 +2,12 @@ import { useReactFlow } from "@xyflow/react";import { HelpTooltip } from "@/components/HelpTooltip";import { ModelSelector } from "@/components/ModelSelector";+import {+ Accordion,+ AccordionContent,+ AccordionItem,+ AccordionTrigger,+} from "@/components/ui/accordion";import { Label } from "@/components/ui/label";import { WorkflowBlockInput } from "@/components/WorkflowBlockInput";import { WorkflowDataSchemaInputGroup } from "@/components/DataSchemaInputGroup/WorkflowDataSchemaInputGroup";@@ -57,15 +63,26 @@ function PDFParserEditorBody({value={model}onChange={(v) => update({ model: v })}/>- <IgnoreWorkflowSystemPrompt- ignoreWorkflowSystemPrompt={- node.data.ignoreWorkflowSystemPrompt ?? false- }- editable={editable}- onIgnoreWorkflowSystemPromptChange={(ignoreWorkflowSystemPrompt) => {- update({ ignoreWorkflowSystemPrompt });- }}- />+ <Accordion type="single" collapsible>+ <AccordionItem value="advanced" className="border-b-0">+ <AccordionTrigger className="py-0">+ Advanced Settings+ </AccordionTrigger>+ <AccordionContent className="pl-6 pr-1 pt-1">+ <IgnoreWorkflowSystemPrompt+ ignoreWorkflowSystemPrompt={+ node.data.ignoreWorkflowSystemPrompt ?? false+ }+ editable={editable}+ onIgnoreWorkflowSystemPromptChange={(+ ignoreWorkflowSystemPrompt,+ ) => {+ update({ ignoreWorkflowSystemPrompt });+ }}+ />+ </AccordionContent>+ </AccordionItem>+ </Accordion></div>);}diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/PdfFillNode/PdfFillEditor.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/PdfFillNode/PdfFillEditor.tsxindex 1cc071b35..f723dca0b 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/PdfFillNode/PdfFillEditor.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/PdfFillNode/PdfFillEditor.tsx@@ -152,13 +152,6 @@ function PdfFillEditorBody({value={data.model}onChange={(value) => update({ model: value })}/>- <IgnoreWorkflowSystemPrompt- ignoreWorkflowSystemPrompt={data.ignoreWorkflowSystemPrompt ?? false}- editable={editable}- onIgnoreWorkflowSystemPromptChange={(ignoreWorkflowSystemPrompt) => {- update({ ignoreWorkflowSystemPrompt });- }}- /><Accordion type="single" collapsible><AccordionItem value="advanced" className="border-b-0">@@ -174,6 +167,17 @@ function PdfFillEditorBody({update({ parameterKeys });}}/>+ <IgnoreWorkflowSystemPrompt+ ignoreWorkflowSystemPrompt={+ data.ignoreWorkflowSystemPrompt ?? false+ }+ editable={editable}+ onIgnoreWorkflowSystemPromptChange={(+ ignoreWorkflowSystemPrompt,+ ) => {+ update({ ignoreWorkflowSystemPrompt });+ }}+ /><div className="space-y-2"><div className="flex gap-2"><Label className="text-xs text-slate-300">LLM Key</Label>diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptEditor.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptEditor.tsxindex c5aedccc4..fff285d4c 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptEditor.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptEditor.tsx@@ -2,6 +2,12 @@ import { useReactFlow } from "@xyflow/react";import { HelpTooltip } from "@/components/HelpTooltip";import { ModelSelector } from "@/components/ModelSelector";+import {+ Accordion,+ AccordionContent,+ AccordionItem,+ AccordionTrigger,+} from "@/components/ui/accordion";import { Label } from "@/components/ui/label";import { Separator } from "@/components/ui/separator";import { WorkflowBlockInputTextarea } from "@/components/WorkflowBlockInputTextarea";@@ -66,15 +72,26 @@ function TextPromptEditorBody({onChange={(next) => update({ jsonSchema: next })}suggestionContext={{ current_schema: jsonSchema }}/>- <IgnoreWorkflowSystemPrompt- ignoreWorkflowSystemPrompt={- node.data.ignoreWorkflowSystemPrompt ?? false- }- editable={editable}- onIgnoreWorkflowSystemPromptChange={(ignoreWorkflowSystemPrompt) => {- update({ ignoreWorkflowSystemPrompt });- }}- />+ <Accordion type="single" collapsible>+ <AccordionItem value="advanced" className="border-b-0">+ <AccordionTrigger className="py-0">+ Advanced Settings+ </AccordionTrigger>+ <AccordionContent className="pl-6 pr-1 pt-1">+ <IgnoreWorkflowSystemPrompt+ ignoreWorkflowSystemPrompt={+ node.data.ignoreWorkflowSystemPrompt ?? false+ }+ editable={editable}+ onIgnoreWorkflowSystemPromptChange={(+ ignoreWorkflowSystemPrompt,+ ) => {+ update({ ignoreWorkflowSystemPrompt });+ }}+ />+ </AccordionContent>+ </AccordionItem>+ </Accordion></div>);}diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/components/BlockParametersDialog.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/components/BlockParametersDialog.tsxindex fa262f7ae..8aa622c6d 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/components/BlockParametersDialog.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/components/BlockParametersDialog.tsx@@ -14,6 +14,7 @@ import { ScrollArea, ScrollAreaViewport } from "@/components/ui/scroll-area";import { WorkflowParameterInput } from "@/routes/workflows/WorkflowParameterInput";import { getLabelForWorkflowParameterType } from "@/routes/workflows/editor/workflowEditorUtils";import type { WorkflowParameter } from "@/routes/workflows/types/workflowTypes";+import { getInvalidJsonMessage } from "@/util/jsonParseError";/*** Validates a parameter value based on its type.@@ -34,8 +35,7 @@ function validateParameterValue(value: unknown, type: string): string | null {JSON.parse(trimmed);return null;} catch (e) {- const message = e instanceof SyntaxError ? e.message : "Parse error";- return `Invalid JSON: ${message}`;+ return getInvalidJsonMessage(trimmed, e);}}return null;diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsxindex 36928acbb..ee3b54f95 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx@@ -66,6 +66,7 @@ import {useWorkflowSettingsStore,type WorkflowSettingsState,} from "@/store/WorkflowSettingsStore";+import { getJsonParseErrorDetail } from "@/util/jsonParseError";import { cn, formatDate, toDate } from "@/util/utils";import {statusIsAFailureType,@@ -188,7 +189,10 @@ const getPayload = (opts: {toast({variant: "warning",title: "Extra HTTP Headers",- description: "Invalid extra HTTP Headers JSON",+ description: `Invalid extra HTTP Headers JSON: ${getJsonParseErrorDetail(+ String(opts.workflowSettings.extraHttpHeaders ?? ""),+ e,+ )}`,});}diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/FileParserBlockForm.test.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/FileParserBlockForm.test.tsxindex f3d553d51..211fda8b9 100644--- a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/FileParserBlockForm.test.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/FileParserBlockForm.test.tsx@@ -159,6 +159,13 @@ describe("FileParserBlockForm (SKY-9381)", () => {expect(screen.getByText("Auto detect")).toBeDefined();expect(screen.getByTestId("json-schema-input")).toBeDefined();expect(screen.getByTestId("model-selector")).toBeDefined();+ const advancedSettings = screen.getByText("Advanced Settings");+ fireEvent.click(advancedSettings);+ expect(+ advancedSettings.compareDocumentPosition(+ screen.getByText("Ignore System Prompt"),+ ) & Node.DOCUMENT_POSITION_FOLLOWING,+ ).toBeTruthy();});test("File URL onChange dispatches updateNodeData via useUpdate (byte-identical write)", () => {diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/PDFParserBlockForm.test.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/PDFParserBlockForm.test.tsxindex 3ab3b1d61..f950cb8b5 100644--- a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/PDFParserBlockForm.test.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/PDFParserBlockForm.test.tsx@@ -154,6 +154,13 @@ describe("PDFParserBlockForm (SKY-93XX)", () => {expect((screen.getByTestId("model-select") as HTMLSelectElement).value,).toBe("gpt-4o");+ const advancedSettings = screen.getByText("Advanced Settings");+ fireEvent.click(advancedSettings);+ expect(+ advancedSettings.compareDocumentPosition(+ screen.getByText("Ignore System Prompt"),+ ) & Node.DOCUMENT_POSITION_FOLLOWING,+ ).toBeTruthy();});test("editing fileUrl propagates via updateNodeData", () => {diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/PdfFillBlockForm.test.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/PdfFillBlockForm.test.tsxindex 8157ad71e..98d0e6960 100644--- a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/PdfFillBlockForm.test.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/PdfFillBlockForm.test.tsx@@ -217,6 +217,13 @@ describe("PdfFillBlockForm", () => {expect(screen.getByText("Prompt")).toBeDefined();expect(screen.getByText("Payload")).toBeDefined();expect(screen.queryByText("API Key")).toBeNull();+ expect(+ screen+ .getByText("Advanced Settings")+ .compareDocumentPosition(+ screen.getByTestId("ignore-workflow-system-prompt"),+ ) & Node.DOCUMENT_POSITION_FOLLOWING,+ ).toBeTruthy();});test("editing file URL, prompt, and payload propagates", () => {diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/TextPromptBlockForm.test.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/TextPromptBlockForm.test.tsxindex 9ea231287..7e0a83c2d 100644--- a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/TextPromptBlockForm.test.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/TextPromptBlockForm.test.tsx@@ -158,6 +158,13 @@ describe("TextPromptBlockForm (SKY-9378)", () => {expect(screen.getByTestId("model-selector").getAttribute("data-value"),).toBe("gpt-4");+ const advancedSettings = screen.getByText("Advanced Settings");+ fireEvent.click(advancedSettings);+ expect(+ advancedSettings.compareDocumentPosition(+ screen.getByText("Ignore System Prompt"),+ ) & Node.DOCUMENT_POSITION_FOLLOWING,+ ).toBeTruthy();});test("persists prompt edits immediately via updateNodeData", () => {diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowParameterEditPanel.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowParameterEditPanel.tsxindex 33c07bb55..54527490a 100644--- a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowParameterEditPanel.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowParameterEditPanel.tsx@@ -38,6 +38,7 @@ import { getDefaultValueForParameterType } from "../workflowEditorUtils";import { validateBitwardenLoginCredential } from "./util";import { HelpTooltip } from "@/components/HelpTooltip";import { useCustomCredentialServiceConfig } from "@/hooks/useCustomCredentialServiceConfig";+import { getInvalidJsonMessage } from "@/util/jsonParseError";import {CredentialDataType,CredentialSource,@@ -1040,32 +1041,28 @@ function WorkflowParameterEditPanel({// Handle workflow parameters (non-credential)if (type === "workflow" && !isCredentialSelected) {+ let defaultValue = defaultValueState.defaultValue;+if (parameterType === "json" &&typeof defaultValueState.defaultValue === "string") {try {- JSON.parse(defaultValueState.defaultValue);+ defaultValue = JSON.parse(defaultValueState.defaultValue);} catch (e) {toast({variant: "destructive",title: "Failed to save input",- description: "Invalid JSON for default value",+ description: getInvalidJsonMessage(+ defaultValueState.defaultValue,+ e,+ ),});return;}}- let defaultValue = defaultValueState.defaultValue;-- // Handle JSON parsing- if (- parameterType === "json" &&- typeof defaultValueState.defaultValue === "string"- ) {- defaultValue = JSON.parse(defaultValueState.defaultValue);- }// Convert boolean to string for backend storage- else if (+ if (parameterType === "boolean" &&typeof defaultValueState.defaultValue === "boolean") {diff --git a/skyvern-frontend/src/routes/workflows/editor/validateErrorCodeMapping.test.ts b/skyvern-frontend/src/routes/workflows/editor/validateErrorCodeMapping.test.tsindex 6e623adbc..2e88bc3ed 100644--- a/skyvern-frontend/src/routes/workflows/editor/validateErrorCodeMapping.test.ts+++ b/skyvern-frontend/src/routes/workflows/editor/validateErrorCodeMapping.test.ts@@ -13,7 +13,7 @@ describe("validateErrorCodeMapping", () => {test("invalid JSON returns a parse error", () => {const result = validateErrorCodeMapping("block_1", "{not json");expect(result).toHaveLength(1);- expect(result[0]).toContain("Error messages is not valid JSON");+ expect(result[0]).toContain("Error messages are not valid JSON");});test("null value (mapping disabled) returns no errors", () => {diff --git a/skyvern-frontend/src/routes/workflows/editor/validateErrorCodeMapping.ts b/skyvern-frontend/src/routes/workflows/editor/validateErrorCodeMapping.tsindex b30ef85c2..c750eab72 100644--- a/skyvern-frontend/src/routes/workflows/editor/validateErrorCodeMapping.ts+++ b/skyvern-frontend/src/routes/workflows/editor/validateErrorCodeMapping.ts@@ -1,3 +1,5 @@+import { getJsonParseErrorDetail } from "@/util/jsonParseError";+/*** Validate an errorCodeMapping JSON string. The value must:* - parse as JSON,@@ -17,8 +19,13 @@ export function validateErrorCodeMapping(let parsed: unknown;try {parsed = JSON.parse(errorCodeMapping);- } catch {- errors.push(`${label}: Error messages is not valid JSON.`);+ } catch (error) {+ errors.push(+ `${label}: Error messages are not valid JSON: ${getJsonParseErrorDetail(+ errorCodeMapping,+ error,+ )}`,+ );return errors;}// `null` is the disabled sentinel — valid.diff --git a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.tsindex 3888d27f5..bec9969fe 100644--- a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts+++ b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts@@ -3,6 +3,7 @@ import { type Node, Edge } from "@xyflow/react";import { nanoid } from "nanoid";import { TSON } from "@/util/tson";+import { getJsonParseErrorDetail } from "@/util/jsonParseError";import {WorkflowBlockType,@@ -4603,7 +4604,10 @@ function getWorkflowErrors(nodes: Array<AppNode>): Array<string> {if (!result.success) {errors.push(- `${node.data.label}: Data schema has invalid templated JSON: ${result.error ?? "-"}`,+ `${node.data.label}: Data schema has invalid templated JSON: ${getJsonParseErrorDetail(+ node.data.dataSchema,+ result.error ?? "Parse error",+ )}`,);}}@@ -4689,7 +4693,10 @@ function getWorkflowErrors(nodes: Array<AppNode>): Array<string> {if (!result.success) {errors.push(- `${node.data.label}: Data schema has invalid templated JSON: ${result.error ?? "-"}`,+ `${node.data.label}: Data schema has invalid templated JSON: ${getJsonParseErrorDetail(+ node.data.dataSchema,+ result.error ?? "Parse error",+ )}`,);}}@@ -4699,8 +4706,13 @@ function getWorkflowErrors(nodes: Array<AppNode>): Array<string> {textPromptNodes.forEach((node) => {try {JSON.parse(node.data.jsonSchema);- } catch {- errors.push(`${node.data.label}: Data schema is not valid JSON.`);+ } catch (error) {+ errors.push(+ `${node.data.label}: Data schema is not valid JSON: ${getJsonParseErrorDetail(+ node.data.jsonSchema,+ error,+ )}`,+ );}});@@ -4708,8 +4720,13 @@ function getWorkflowErrors(nodes: Array<AppNode>): Array<string> {pdfParserNodes.forEach((node) => {try {JSON.parse(node.data.jsonSchema);- } catch {- errors.push(`${node.data.label}: Data schema is not valid JSON.`);+ } catch (error) {+ errors.push(+ `${node.data.label}: Data schema is not valid JSON: ${getJsonParseErrorDetail(+ node.data.jsonSchema,+ error,+ )}`,+ );}});@@ -4717,8 +4734,13 @@ function getWorkflowErrors(nodes: Array<AppNode>): Array<string> {fileParserNodes.forEach((node) => {try {JSON.parse(node.data.jsonSchema);- } catch {- errors.push(`${node.data.label}: Data schema is not valid JSON.`);+ } catch (error) {+ errors.push(+ `${node.data.label}: Data schema is not valid JSON: ${getJsonParseErrorDetail(+ node.data.jsonSchema,+ error,+ )}`,+ );}});diff --git a/skyvern-frontend/src/store/WorkflowHasChangesStore.ts b/skyvern-frontend/src/store/WorkflowHasChangesStore.tsindex bdcef7a87..c6bd85837 100644--- a/skyvern-frontend/src/store/WorkflowHasChangesStore.ts+++ b/skyvern-frontend/src/store/WorkflowHasChangesStore.ts@@ -8,6 +8,7 @@ import { usePostHog } from "posthog-js/react";import { getClient } from "@/api/AxiosClient";import { toast } from "@/components/ui/use-toast";import { useCredentialGetter } from "@/hooks/useCredentialGetter";+import { getJsonParseErrorDetail } from "@/util/jsonParseError";import {type BlockYAML,type ParameterYAML,@@ -131,7 +132,10 @@ const useWorkflowSave = (opts?: WorkflowSaveOpts) => {} catch (error) {toast({title: "Error",- description: "Invalid JSON format in extra http headers",+ description: `Invalid JSON format in extra http headers: ${getJsonParseErrorDetail(+ saveData.settings.extraHttpHeaders ?? "",+ error,+ )}`,variant: "destructive",});return;@@ -163,7 +167,10 @@ const useWorkflowSave = (opts?: WorkflowSaveOpts) => {} catch (error) {toast({title: "Error",- description: "Invalid JSON format in cdp connect headers",+ description: `Invalid JSON format in cdp connect headers: ${getJsonParseErrorDetail(+ saveData.settings.cdpConnectHeaders ?? "",+ error,+ )}`,variant: "destructive",});return;diff --git a/skyvern-frontend/src/util/jsonParseError.test.ts b/skyvern-frontend/src/util/jsonParseError.test.tsnew file mode 100644index 000000000..f175f8892--- /dev/null+++ b/skyvern-frontend/src/util/jsonParseError.test.ts@@ -0,0 +1,60 @@+import { describe, expect, test } from "vitest";++import {+ getInvalidJsonMessage,+ getJsonParseErrorDetail,+} from "./jsonParseError";++describe("json parse error formatting", () => {+ test("explains hidden non-breaking spaces outside strings", () => {+ const value = `{+ "personalInfo": {+ "firstName": "Luis",+ "lastName": "Ortiz"\u00a0+ }+}`;+ let error: unknown;++ try {+ JSON.parse(value);+ } catch (caught) {+ error = caught;+ }++ expect(getInvalidJsonMessage(value, error)).toContain(+ "Invalid JSON: JSON contains a non-breaking space (U+00A0) at line 4, column 24; replace it with a regular space.",+ );+ });++ test("preserves parser details for ordinary JSON syntax errors", () => {+ const value = '{"first": "Luis",}';+ let error: unknown;++ try {+ JSON.parse(value);+ } catch (caught) {+ error = caught;+ }++ expect(getJsonParseErrorDetail(value, error)).toMatch(+ /Expected double-quoted property name|Unexpected token/,+ );+ });++ test("adds line and column when the parser only reports a position", () => {+ const value = '{\n "first": "Luis",\n}';++ expect(+ getJsonParseErrorDetail(+ value,+ "Unexpected token } in JSON at position 21",+ ),+ ).toBe("Unexpected token } in JSON at position 21 (line 3 column 1)");+ });++ test("does not flag non-breaking spaces inside strings", () => {+ const value = '{"lastName": "Ortiz\u00a0"}';++ expect(getJsonParseErrorDetail(value, "Parse error")).toBe("Parse error");+ });+});diff --git a/skyvern-frontend/src/util/jsonParseError.ts b/skyvern-frontend/src/util/jsonParseError.tsnew file mode 100644index 000000000..c4a67303d--- /dev/null+++ b/skyvern-frontend/src/util/jsonParseError.ts@@ -0,0 +1,133 @@+type JsonLocation = {+ position: number;+ line: number;+ column: number;+};++const allowedJsonWhitespace = new Set([" ", "\n", "\r", "\t"]);++const unsupportedWhitespaceNames: Record<string, string> = {+ "\u00a0": "non-breaking space",+ "\u1680": "ogham space mark",+ "\u2000": "en quad",+ "\u2001": "em quad",+ "\u2002": "en space",+ "\u2003": "em space",+ "\u2004": "three-per-em space",+ "\u2005": "four-per-em space",+ "\u2006": "six-per-em space",+ "\u2007": "figure space",+ "\u2008": "punctuation space",+ "\u2009": "thin space",+ "\u200a": "hair space",+ "\u2028": "line separator",+ "\u2029": "paragraph separator",+ "\u202f": "narrow non-breaking space",+ "\u205f": "medium mathematical space",+ "\u3000": "ideographic space",+ "\ufeff": "byte order mark",+};++function getLocation(input: string, position: number): JsonLocation {+ let line = 1;+ let column = 1;++ for (let i = 0; i < position; i++) {+ if (input.charAt(i) === "\n") {+ line++;+ column = 1;+ } else {+ column++;+ }+ }++ return { position, line, column };+}++function getCodePointLabel(char: string): string {+ const codePoint = char.codePointAt(0) ?? char.charCodeAt(0);+ return `U+${codePoint.toString(16).toUpperCase().padStart(4, "0")}`;+}++function getErrorMessage(error: unknown): string {+ if (error instanceof Error) {+ return error.message;+ }+ if (typeof error === "string") {+ return error;+ }+ return String(error);+}++function getUnsupportedWhitespace(input: string): {+ char: string;+ location: JsonLocation;+} | null {+ let inString = false;+ let escapeNext = false;++ for (let i = 0; i < input.length; i++) {+ const char = input.charAt(i);++ if (escapeNext) {+ escapeNext = false;+ continue;+ }++ if (char === "\\") {+ escapeNext = true;+ continue;+ }++ if (char === '"') {+ inString = !inString;+ continue;+ }++ if (!inString && !allowedJsonWhitespace.has(char) && /\s/u.test(char)) {+ return { char, location: getLocation(input, i) };+ }+ }++ return null;+}++function addLocationFromPosition(input: string, message: string): string {+ if (/line \d+ column \d+/i.test(message)) {+ return message;+ }++ const positionMatch = message.match(/position (\d+)/i);+ if (!positionMatch) {+ return message;+ }++ const position = Number(positionMatch[1]);+ if (!Number.isFinite(position)) {+ return message;+ }++ const { line, column } = getLocation(input, position);+ return `${message} (line ${line} column ${column})`;+}++export function getJsonParseErrorDetail(input: string, error: unknown): string {+ const message = getErrorMessage(error) || "Parse error";+ const unsupportedWhitespace = getUnsupportedWhitespace(input);++ if (unsupportedWhitespace) {+ const name =+ unsupportedWhitespaceNames[unsupportedWhitespace.char] ??+ "unsupported whitespace";+ const codePoint = getCodePointLabel(unsupportedWhitespace.char);+ const { line, column } = unsupportedWhitespace.location;++ return `JSON contains a ${name} (${codePoint}) at line ${line}, column ${column}; replace it with a regular space. Parser detail: ${addLocationFromPosition(input, message)}`;+ }++ return addLocationFromPosition(input, message);+}++export function getInvalidJsonMessage(input: string, error: unknown): string {+ return `Invalid JSON: ${getJsonParseErrorDetail(input, error)}`;+}diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex f3018fb0e..b011fb000 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -33,6 +33,8 @@ from charset_normalizer import from_bytesfrom email_validator import EmailNotValidError, validate_emailfrom jinja2 import StrictUndefined, TemplateSyntaxErrorfrom jinja2.sandbox import SandboxedEnvironment+from jsonschema import Draft202012Validator+from jsonschema.exceptions import ValidationErrorfrom opentelemetry import trace as otel_tracefrom playwright.async_api import Pagefrom pydantic import BaseModel, Field, model_validator@@ -83,6 +85,8 @@ from skyvern.forge.sdk.api.files import (from skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandlerfrom skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactoryfrom skyvern.forge.sdk.api.llm.custom_llm_registry import is_custom_llm_model_name+from skyvern.forge.sdk.api.llm.exceptions import InvalidLLMResponseFormat, InvalidLLMResponseType+from skyvern.forge.sdk.api.llm.schema_validator import validate_schemafrom skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.aiohttp_helper import aiohttp_request@@ -4378,6 +4382,173 @@ async def wrapper({default_args}):)+SCHEMA_VALIDATION_MAX_ATTEMPTS = 2+SCHEMA_VALIDATION_MAX_ERRORS = 5+++def _default_structured_output_schema(description: str) -> dict[str, Any]:+ # The output field is optional to preserve the legacy permissive default schema.+ return {+ "type": "object",+ "properties": {+ "output": {+ "type": "object",+ "description": description,+ }+ },+ }+++def _default_text_prompt_schema() -> dict[str, Any]:+ return {+ "type": "object",+ "properties": {+ "llm_response": {+ "type": "string",+ "description": "Your response to the prompt",+ }+ },+ }+++def _json_type_name(value: Any) -> str:+ if value is None:+ return "null"+ if isinstance(value, bool):+ return "boolean"+ if isinstance(value, dict):+ return "object"+ if isinstance(value, list):+ return "array"+ if isinstance(value, str):+ return "string"+ if isinstance(value, int):+ return "integer"+ if isinstance(value, float):+ return "number"+ return type(value).__name__+++def _schema_type_description(schema_type: Any) -> str:+ if isinstance(schema_type, list):+ return " or ".join(str(t) for t in schema_type)+ return str(schema_type)+++def _schema_path(error: ValidationError) -> str:+ schema_path = list(error.absolute_schema_path)+ path_parts: list[str] = []+ index = 0+ while index < len(schema_path):+ part = schema_path[index]+ if part == "properties" and index + 1 < len(schema_path):+ path_parts.append(str(schema_path[index + 1]))+ index += 2+ continue+ if part == "items":+ path_parts.append("[]")+ index += 1+ continue+ if part == "additionalProperties":+ path_parts.append("<map value>")+ index += 1+ continue+ if part == "patternProperties":+ path_parts.append("<map value>")+ index += 2 if index + 1 < len(schema_path) else 1+ continue+ index += 1++ return "root" + "".join(f"{part}" if part == "[]" else f".{part}" for part in path_parts)+++def _format_schema_validation_error(error: ValidationError) -> str:+ path = _schema_path(error)+ actual_type = _json_type_name(error.instance)++ if error.validator == "type":+ expected_type = _schema_type_description(error.validator_value)+ return f"{path}: expected type {expected_type}, got {actual_type}"++ if error.validator == "required":+ match = re.match(r"'([^']+)' is a required property", error.message)+ if match:+ return f"{path}: missing required property {match.group(1)}"+ return f"{path}: missing required property"++ if error.validator == "additionalProperties":+ unexpected_count: int | None = None+ schema_properties = error.schema.get("properties", {}) if isinstance(error.schema, dict) else {}+ if isinstance(error.instance, dict) and isinstance(schema_properties, dict):+ unexpected_count = sum(1 for field in error.instance if field not in schema_properties)+ if unexpected_count is not None:+ return f"{path}: has {unexpected_count} unexpected properties"+ return f"{path}: has unexpected properties"++ if error.validator in {"minItems", "maxItems"} and isinstance(error.instance, list):+ return f"{path}: violates {error.validator}={error.validator_value}; item count={len(error.instance)}"++ if error.validator in {"minLength", "maxLength"} and isinstance(error.instance, str):+ return f"{path}: violates {error.validator}={error.validator_value}; string length={len(error.instance)}"++ if error.validator == "enum":+ allowed_count = len(error.validator_value) if isinstance(error.validator_value, list) else "configured"+ return f"{path}: value is not one of {allowed_count} allowed values; got {actual_type}"++ return f"{path}: violates {error.validator} constraint; got {actual_type}"+++def _validate_response_against_json_schema(+ response: Any,+ json_schema: dict[str, Any] | None,+ schema_label: str,+ max_errors: int = SCHEMA_VALIDATION_MAX_ERRORS,+) -> str | None:+ if not json_schema:+ return None++ if not validate_schema(json_schema):+ return f"{schema_label} JSON schema is invalid."++ try:+ validator = Draft202012Validator(json_schema)+ validation_errors = [_format_schema_validation_error(error) for error in validator.iter_errors(response)]+ except Exception as e:+ LOG.warning(+ "Failed to validate LLM response against JSON schema",+ schema_label=schema_label,+ error_type=type(e).__name__,+ exc_info=True,+ )+ return f"{schema_label} JSON schema validation failed ({type(e).__name__})."++ validation_errors = list(dict.fromkeys(validation_errors))+ if not validation_errors:+ return None++ return f"LLM response does not match {schema_label.lower()} JSON schema: " + "; ".join(+ validation_errors[:max_errors]+ )+++def _is_schema_configuration_failure(failure_reason: str) -> bool:+ return "JSON schema is invalid" in failure_reason or "JSON schema validation failed" in failure_reason+++def _llm_response_format_failure_reason(error: Exception) -> str:+ return f"LLM response could not be parsed or coerced into the required JSON shape ({type(error).__name__})."+++def _build_schema_validation_retry_prompt(prompt: str, failure_reason: str) -> str:+ return (+ f"{prompt}\n\n"+ "Your previous response failed JSON schema validation.\n"+ f"Validation error: {failure_reason}\n\n"+ "Retry the task. Return only valid JSON that exactly matches the schema. "+ "Do not include markdown, code fences, explanatory text, or extra fields."+ )++class TextPromptBlock(Block):# There is a mypy bug with Literal. Without the type: ignore, mypy will raise an error:# Parameter 1 of Literal[...] cannot be of type "Any"@@ -4387,6 +4558,8 @@ class TextPromptBlock(Block):prompt: strparameters: list[PARAMETER_TYPE] = []json_schema: dict[str, Any] | None = None+ schema_validation_max_attempts: ClassVar[int] = SCHEMA_VALIDATION_MAX_ATTEMPTS+ schema_validation_max_errors: ClassVar[int] = SCHEMA_VALIDATION_MAX_ERRORSdef get_all_parameters(self,@@ -4423,6 +4596,14 @@ class TextPromptBlock(Block):self._apply_workflow_system_prompt(workflow_run_context)+ def _validate_response_against_json_schema(self, response: Any) -> str | None:+ return _validate_response_against_json_schema(+ response,+ self.json_schema,+ "Text prompt",+ max_errors=self.schema_validation_max_errors,+ )+async def send_prompt(self,prompt: str,@@ -4430,28 +4611,23 @@ class TextPromptBlock(Block):workflow_run_id: str,organization_id: str | None = None,workflow_run_block_id: str | None = None,- ) -> dict[str, Any]:+ schema_validation_failure: str | None = None,+ json_schema: dict[str, Any] | None = None,+ ) -> dict[str, Any] | list | str | None:default_llm_handler = await self._resolve_default_llm_handler(workflow_run_id, organization_id)llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(self.override_llm_key_for_organization(organization_id) or self.llm_key, default=default_llm_handler)- if not self.json_schema:- self.json_schema = {- "type": "object",- "properties": {- "llm_response": {- "type": "string",- "description": "Your response to the prompt",- }- },- }+ schema_to_use = json_schema or self.json_schema or _default_text_prompt_schema()prompt = prompt_engine.load_prompt_from_string(prompt, **parameter_values)+ if schema_validation_failure:+ prompt = _build_schema_validation_retry_prompt(prompt, schema_validation_failure)prompt += ("\n\n"+ "Please respond to the prompt above using the following JSON definition:\n\n"+ "```json\n"- + json.dumps(self.json_schema, indent=2)+ + json.dumps(schema_to_use, indent=2)+ "\n```\n\n")@@ -4478,6 +4654,8 @@ class TextPromptBlock(Block):system_prompt=self.workflow_system_prompt,workflow_run_block_id=workflow_run_block_id,organization_id=organization_id,+ # Schema validation must inspect the raw parsed root; dict coercion can hide wrong-root responses.+ force_dict=False,)if workflow_run_block:@@ -4570,33 +4748,111 @@ class TextPromptBlock(Block):else:parameter_values[parameter.key] = value- try:- response = await self.send_prompt(- self.prompt,- parameter_values,- workflow_run_id,- organization_id,- workflow_run_block_id=workflow_run_block_id,- )- except Exception as e:- try:- resolved_llm_key = self.override_llm_key_for_organization(organization_id) or self.llm_key- except Exception:- resolved_llm_key = self.llm_key- LOG.exception(- "TextPromptBlock LLM call failed",- block_label=self.label,- workflow_run_id=workflow_run_id,- llm_key=resolved_llm_key,- )+ response: dict[str, Any] | list | str | None = None+ schema_to_use = self.json_schema or _default_text_prompt_schema()+ if not validate_schema(schema_to_use):return await self.build_block_result(success=False,- failure_reason=f"LLM call failed: {e}",+ failure_reason="Text prompt JSON schema is invalid.",output_parameter_value=None,status=BlockStatus.failed,workflow_run_block_id=workflow_run_block_id,organization_id=organization_id,)++ schema_validation_failure_for_retry: str | None = None+ for attempt in range(self.schema_validation_max_attempts):+ try:+ response = await self.send_prompt(+ self.prompt,+ parameter_values,+ workflow_run_id,+ organization_id,+ workflow_run_block_id=workflow_run_block_id,+ schema_validation_failure=schema_validation_failure_for_retry,+ json_schema=schema_to_use,+ )+ except (InvalidLLMResponseFormat, InvalidLLMResponseType) as e:+ response_format_failure_reason = _llm_response_format_failure_reason(e)+ will_retry = attempt + 1 < self.schema_validation_max_attempts+ LOG.warning(+ "TextPromptBlock LLM response failed response-format validation",+ block_label=self.label,+ workflow_run_id=workflow_run_id,+ workflow_run_block_id=workflow_run_block_id,+ attempt=attempt + 1,+ max_attempts=self.schema_validation_max_attempts,+ will_retry=will_retry,+ error_type=type(e).__name__,+ schema_type=schema_to_use.get("type"),+ )+ if not will_retry:+ return await self.build_block_result(+ success=False,+ failure_reason=response_format_failure_reason,+ output_parameter_value=None,+ status=BlockStatus.failed,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ )++ schema_validation_failure_for_retry = response_format_failure_reason+ continue+ except Exception as e:+ try:+ resolved_llm_key = self.override_llm_key_for_organization(organization_id) or self.llm_key+ except Exception:+ resolved_llm_key = self.llm_key+ LOG.exception(+ "TextPromptBlock LLM call failed",+ block_label=self.label,+ workflow_run_id=workflow_run_id,+ llm_key=resolved_llm_key,+ )+ return await self.build_block_result(+ success=False,+ failure_reason=f"LLM call failed: {e}",+ output_parameter_value=None,+ status=BlockStatus.failed,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ )++ schema_validation_failure = _validate_response_against_json_schema(+ response,+ schema_to_use,+ "Text prompt",+ max_errors=self.schema_validation_max_errors,+ )+ if not schema_validation_failure:+ break++ is_schema_configuration_failure = _is_schema_configuration_failure(schema_validation_failure)+ will_retry = attempt + 1 < self.schema_validation_max_attempts and not is_schema_configuration_failure+ LOG.warning(+ "TextPromptBlock LLM response failed schema validation",+ block_label=self.label,+ workflow_run_id=workflow_run_id,+ workflow_run_block_id=workflow_run_block_id,+ attempt=attempt + 1,+ max_attempts=self.schema_validation_max_attempts,+ will_retry=will_retry,+ failure_reason=schema_validation_failure,+ schema_type=schema_to_use.get("type"),+ )+ if not will_retry:+ return await self.build_block_result(+ success=False,+ failure_reason=schema_validation_failure,+ output_parameter_value=None,+ status=BlockStatus.failed,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ )++ schema_validation_failure_for_retry = schema_validation_failure+ continue+await self.record_output_parameter_value(workflow_run_context, workflow_run_id, response)return await self.build_block_result(success=True,@@ -5789,6 +6045,8 @@ class FileParserBlock(Block):file_url: strfile_type: FileType = FileType.AUTO_DETECTjson_schema: dict[str, Any] | None = None+ schema_validation_max_attempts: ClassVar[int] = SCHEMA_VALIDATION_MAX_ATTEMPTS+ ocr_validation_max_attempts: ClassVar[int] = SCHEMA_VALIDATION_MAX_ATTEMPTSdef get_failure_error_codes(self) -> list[str]:return ["FILE_PARSER_ERROR"]@@ -5809,6 +6067,38 @@ class FileParserBlock(Block):self._apply_workflow_system_prompt(workflow_run_context)+ @staticmethod+ def _validate_ocr_llm_response(llm_response: Any) -> str | None:+ if not isinstance(llm_response, dict):+ return (+ f"OCR response must be a JSON object with extracted_text string; got {_json_type_name(llm_response)}."+ )+ if not isinstance(llm_response.get("extracted_text"), str):+ return (+ "OCR response must include extracted_text as a string; "+ f"got {_json_type_name(llm_response.get('extracted_text'))}."+ )+ return None++ @staticmethod+ def _build_ocr_validation_retry_prompt(prompt: str, failure_reason: str) -> str:+ return (+ f"{prompt}\n\n"+ "Your previous OCR response failed JSON validation.\n"+ f"Validation error: {failure_reason}\n\n"+ 'Retry the task. Return only valid JSON with this exact shape: {"extracted_text": "..."} '+ "Do not include markdown, code fences, explanatory text, or extra fields."+ )++ @staticmethod+ def _validate_ai_response_against_json_schema(response: Any, json_schema: dict[str, Any]) -> str | None:+ return _validate_response_against_json_schema(+ response,+ json_schema,+ "File parser",+ max_errors=SCHEMA_VALIDATION_MAX_ERRORS,+ )+def _detect_file_type_from_url(self, file_url: str, file_path: str | None = None) -> FileType:"""Detect file type based on file extension in the URL, with magic-byte fallback."""url_parsed = urlparse(file_url)@@ -6089,6 +6379,9 @@ class FileParserBlock(Block):bounded concurrency, reassembled in page order with page markers, andtruncated at a page boundary once MAX_FILE_PARSE_INPUT_TOKENS is reached."""+ if self.ocr_validation_max_attempts <= 0:+ raise ValueError("OCR validation max attempts must be greater than 0.")+llm_prompt = prompt_engine.load_prompt("extract-text-from-image")default_handler = await self._resolve_file_parser_handler("extract-text-from-image", workflow_run_block_id, organization_id@@ -6100,17 +6393,53 @@ class FileParserBlock(Block):async def _ocr_page(page_image: bytes) -> str:async with semaphore:- # OCR transcription intentionally skips system_prompt; it still applies- # to the downstream extract-information-from-file-text call.- llm_response = await llm_api_handler(- prompt=llm_prompt,- prompt_name="extract-text-from-image",- screenshots=[page_image],- force_dict=True,- workflow_run_block_id=workflow_run_block_id,- organization_id=organization_id,- )- return llm_response.get("extracted_text", "") or ""+ prompt_for_attempt = llm_prompt+ for attempt in range(self.ocr_validation_max_attempts):+ try:+ # OCR transcription intentionally skips system_prompt; it still applies+ # to the downstream extract-information-from-file-text call.+ llm_response = await llm_api_handler(+ prompt=prompt_for_attempt,+ prompt_name="extract-text-from-image",+ screenshots=[page_image],+ # Schema validation must inspect the raw parsed root; dict coercion can hide bad OCR JSON.+ force_dict=False,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ )+ except (InvalidLLMResponseFormat, InvalidLLMResponseType) as e:+ failure_reason = _llm_response_format_failure_reason(e)+ will_retry = attempt + 1 < self.ocr_validation_max_attempts+ LOG.warning(+ "FileParserBlock PDF OCR LLM response failed response-format validation",+ file_url=self.file_url,+ attempt=attempt + 1,+ max_attempts=self.ocr_validation_max_attempts,+ will_retry=will_retry,+ error_type=type(e).__name__,+ )+ if not will_retry:+ raise ValueError(failure_reason) from e+ prompt_for_attempt = self._build_ocr_validation_retry_prompt(llm_prompt, failure_reason)+ continue++ ocr_failure_reason = self._validate_ocr_llm_response(llm_response)+ if not ocr_failure_reason:+ return llm_response.get("extracted_text", "") or ""++ will_retry = attempt + 1 < self.ocr_validation_max_attempts+ LOG.warning(+ "FileParserBlock PDF OCR LLM response failed schema validation",+ file_url=self.file_url,+ attempt=attempt + 1,+ max_attempts=self.ocr_validation_max_attempts,+ will_retry=will_retry,+ failure_reason=ocr_failure_reason,+ )+ if not will_retry:+ raise ValueError(ocr_failure_reason)+ prompt_for_attempt = self._build_ocr_validation_retry_prompt(llm_prompt, ocr_failure_reason)+ raise RuntimeError("OCR retry loop exhausted without returning or raising.")page_results = await asyncio.gather(*(_ocr_page(page_image) for page_image in page_images),@@ -6160,6 +6489,9 @@ class FileParserBlock(Block):organization_id: str | None = None,) -> str:"""Parse image file using vision LLM for OCR."""+ if self.ocr_validation_max_attempts <= 0:+ raise ValueError("OCR validation max attempts must be greater than 0.")+try:with open(file_path, "rb") as f:image_bytes = f.read()@@ -6173,15 +6505,52 @@ class FileParserBlock(Block):)# OCR transcription intentionally skips system_prompt — see# _parse_pdf_file_with_vision_ocr for rationale.- llm_response = await llm_api_handler(- prompt=llm_prompt,- prompt_name="extract-text-from-image",- screenshots=[image_bytes],- force_dict=True,- workflow_run_block_id=workflow_run_block_id,- organization_id=organization_id,- )- return llm_response.get("extracted_text", "")+ prompt_for_attempt = llm_prompt+ for attempt in range(self.ocr_validation_max_attempts):+ try:+ llm_response = await llm_api_handler(+ prompt=prompt_for_attempt,+ prompt_name="extract-text-from-image",+ screenshots=[image_bytes],+ # Schema validation must inspect the raw parsed root; dict coercion can hide bad OCR JSON.+ force_dict=False,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ )+ except (InvalidLLMResponseFormat, InvalidLLMResponseType) as e:+ failure_reason = _llm_response_format_failure_reason(e)+ will_retry = attempt + 1 < self.ocr_validation_max_attempts+ LOG.warning(+ "FileParserBlock image OCR LLM response failed response-format validation",+ file_url=self.file_url,+ attempt=attempt + 1,+ max_attempts=self.ocr_validation_max_attempts,+ will_retry=will_retry,+ error_type=type(e).__name__,+ )+ if not will_retry:+ raise ValueError(failure_reason) from e+ prompt_for_attempt = self._build_ocr_validation_retry_prompt(llm_prompt, failure_reason)+ continue++ ocr_failure_reason = self._validate_ocr_llm_response(llm_response)+ if not ocr_failure_reason:+ return llm_response.get("extracted_text", "") or ""++ will_retry = attempt + 1 < self.ocr_validation_max_attempts+ LOG.warning(+ "FileParserBlock image OCR LLM response failed schema validation",+ file_url=self.file_url,+ attempt=attempt + 1,+ max_attempts=self.ocr_validation_max_attempts,+ will_retry=will_retry,+ failure_reason=ocr_failure_reason,+ )+ if not will_retry:+ raise ValueError(ocr_failure_reason)+ prompt_for_attempt = self._build_ocr_validation_retry_prompt(llm_prompt, ocr_failure_reason)++ raise RuntimeError("OCR retry loop exhausted without returning or raising.")except Exception:LOG.exception("Failed to extract text from image via OCR", file_url=self.file_url)raise@@ -6262,18 +6631,12 @@ class FileParserBlock(Block):workflow_run_context: WorkflowRunContext,workflow_run_block_id: str | None = None,organization_id: str | None = None,- ) -> dict[str, Any]:+ ) -> dict[str, Any] | list | str | None:"""Extract structured data using AI based on json_schema."""# Use local variable to avoid mutating the instance- schema_to_use = self.json_schema or {- "type": "object",- "properties": {- "output": {- "type": "object",- "description": "Information extracted from the file",- }- },- }+ schema_to_use = self.json_schema or _default_structured_output_schema("Information extracted from the file")+ if not validate_schema(schema_to_use):+ raise ValueError("File parser JSON schema is invalid.")# Convert content to string for AI processingif isinstance(content, list):@@ -6291,15 +6654,58 @@ class FileParserBlock(Block):)llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(llm_key, default=default_handler)- llm_response = await llm_api_handler(- prompt=llm_prompt,- prompt_name="extract-information-from-file-text",- force_dict=False,- system_prompt=self.workflow_system_prompt,- workflow_run_block_id=workflow_run_block_id,- organization_id=organization_id,- )- return llm_response+ prompt_for_attempt = llm_prompt+ for attempt in range(self.schema_validation_max_attempts):+ try:+ llm_response = await llm_api_handler(+ prompt=prompt_for_attempt,+ prompt_name="extract-information-from-file-text",+ # Schema validation must inspect the raw parsed root; dict coercion can hide wrong-root responses.+ force_dict=False,+ system_prompt=self.workflow_system_prompt,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ )+ except (InvalidLLMResponseFormat, InvalidLLMResponseType) as e:+ failure_reason = _llm_response_format_failure_reason(e)+ will_retry = attempt + 1 < self.schema_validation_max_attempts+ LOG.warning(+ "FileParserBlock extraction LLM response failed response-format validation",+ file_url=self.file_url,+ attempt=attempt + 1,+ max_attempts=self.schema_validation_max_attempts,+ will_retry=will_retry,+ error_type=type(e).__name__,+ schema_type=schema_to_use.get("type"),+ )+ if not will_retry:+ raise ValueError(failure_reason) from e+ prompt_for_attempt = _build_schema_validation_retry_prompt(llm_prompt, failure_reason)+ continue++ schema_validation_failure = self._validate_ai_response_against_json_schema(llm_response, schema_to_use)+ if not schema_validation_failure:+ return llm_response++ is_schema_configuration_failure = _is_schema_configuration_failure(schema_validation_failure)+ will_retry = attempt + 1 < self.schema_validation_max_attempts and not is_schema_configuration_failure+ LOG.warning(+ "FileParserBlock extraction LLM response failed schema validation",+ file_url=self.file_url,+ attempt=attempt + 1,+ max_attempts=self.schema_validation_max_attempts,+ will_retry=will_retry,+ failure_reason=schema_validation_failure,+ schema_type=schema_to_use.get("type"),+ )+ if not will_retry:+ raise ValueError(schema_validation_failure)+ prompt_for_attempt = _build_schema_validation_retry_prompt(+ llm_prompt,+ schema_validation_failure,+ )++ raise AssertionError("unreachable schema validation retry loop exit")async def _record_failure(self,@@ -6468,7 +6874,7 @@ class FileParserBlock(Block):)# If json_schema is provided, use AI to extract structured data- final_data: str | list[dict[str, Any]] | dict[str, Any]+ final_data: AnyLOG.debug("FileParserBlock JSON schema check",has_json_schema=self.json_schema is not None,@@ -6521,6 +6927,7 @@ class PDFParserBlock(Block):file_url: strjson_schema: dict[str, Any] | None = None+ schema_validation_max_attempts: ClassVar[int] = SCHEMA_VALIDATION_MAX_ATTEMPTSdef get_all_parameters(self,@@ -6589,27 +6996,92 @@ class PDFParserBlock(Block):)if not self.json_schema:- self.json_schema = {- "type": "object",- "properties": {- "output": {- "type": "object",- "description": "Information extracted from the text",- }- },- }+ self.json_schema = _default_structured_output_schema("Information extracted from the text")+ schema_to_use = self.json_schema+ assert schema_to_use is not None+ if not validate_schema(schema_to_use):+ return await self.build_block_result(+ success=False,+ failure_reason="File parser JSON schema is invalid.",+ output_parameter_value=None,+ status=BlockStatus.failed,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ )llm_prompt = prompt_engine.load_prompt(- "extract-information-from-file-text", extracted_text_content=extracted_text, json_schema=self.json_schema- )- llm_response = await app.LLM_API_HANDLER(- prompt=llm_prompt,- prompt_name="extract-information-from-file-text",- force_dict=False,- system_prompt=self.workflow_system_prompt,- workflow_run_block_id=workflow_run_block_id,- organization_id=organization_id,+ "extract-information-from-file-text", extracted_text_content=extracted_text, json_schema=schema_to_use)++ llm_response: dict[str, Any] | list | str | None = None+ prompt_for_attempt = llm_prompt+ for attempt in range(self.schema_validation_max_attempts):+ try:+ llm_response = await app.LLM_API_HANDLER(+ prompt=prompt_for_attempt,+ prompt_name="extract-information-from-file-text",+ # Schema validation must inspect the raw parsed root; dict coercion can hide wrong-root responses.+ force_dict=False,+ system_prompt=self.workflow_system_prompt,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ )+ except (InvalidLLMResponseFormat, InvalidLLMResponseType) as e:+ failure_reason = _llm_response_format_failure_reason(e)+ will_retry = attempt + 1 < self.schema_validation_max_attempts+ LOG.warning(+ "PDFParserBlock extraction LLM response failed response-format validation",+ file_url=self.file_url,+ attempt=attempt + 1,+ max_attempts=self.schema_validation_max_attempts,+ will_retry=will_retry,+ error_type=type(e).__name__,+ schema_type=schema_to_use.get("type"),+ )+ if not will_retry:+ return await self.build_block_result(+ success=False,+ failure_reason=failure_reason,+ output_parameter_value=None,+ status=BlockStatus.failed,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ )+ prompt_for_attempt = _build_schema_validation_retry_prompt(llm_prompt, failure_reason)+ continue++ schema_validation_failure = FileParserBlock._validate_ai_response_against_json_schema(+ llm_response,+ schema_to_use,+ )+ if not schema_validation_failure:+ break++ is_schema_configuration_failure = _is_schema_configuration_failure(schema_validation_failure)+ will_retry = attempt + 1 < self.schema_validation_max_attempts and not is_schema_configuration_failure+ LOG.warning(+ "PDFParserBlock extraction LLM response failed schema validation",+ file_url=self.file_url,+ attempt=attempt + 1,+ max_attempts=self.schema_validation_max_attempts,+ will_retry=will_retry,+ failure_reason=schema_validation_failure,+ schema_type=schema_to_use.get("type"),+ )+ if not will_retry:+ return await self.build_block_result(+ success=False,+ failure_reason=schema_validation_failure,+ output_parameter_value=None,+ status=BlockStatus.failed,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ )+ prompt_for_attempt = _build_schema_validation_retry_prompt(+ llm_prompt,+ schema_validation_failure,+ )+# Record the parsed dataawait self.record_output_parameter_value(workflow_run_context, workflow_run_id, llm_response)return await self.build_block_result(diff --git a/tests/unit/test_text_prompt_block.py b/tests/unit/test_text_prompt_block.pyindex 933ba9d4f..6b65203c1 100644--- a/tests/unit/test_text_prompt_block.py+++ b/tests/unit/test_text_prompt_block.py@@ -9,11 +9,12 @@ from jinja2.sandbox import SandboxedEnvironmentimport skyvern.forge.sdk.workflow.models.block as _block_modfrom skyvern.config import settings as base_settingsfrom skyvern.forge.prompts import prompt_engine+from skyvern.forge.sdk.api.llm.exceptions import InvalidLLMResponseFormatfrom skyvern.forge.sdk.settings_manager import SettingsManagerfrom skyvern.forge.sdk.workflow.context_manager import WorkflowRunContextfrom skyvern.forge.sdk.workflow.models.block import TextPromptBlockfrom skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType-from skyvern.schemas.workflows import TextPromptBlockYAML, WorkflowRequest+from skyvern.schemas.workflows import BlockStatus, TextPromptBlockYAML, WorkflowRequestblock_module = sys.modules["skyvern.forge.sdk.workflow.models.block"]@@ -128,6 +129,7 @@ async def test_text_prompt_block_uses_selected_model(monkeypatch, model_name):assert captured["llm_key"] == expected_llm_keyassert captured["prompt_name"] == "text-prompt"assert response == {"llm_response": "ok"}+ assert block.json_schema is None@pytest.mark.asyncio@@ -401,6 +403,434 @@ async def test_text_prompt_block_uses_explicit_internal_llm_key_override(monkeypassert response == {"llm_response": "override"}+async def test_text_prompt_block_array_schema_does_not_force_dict(monkeypatch):+ block = _make_text_prompt_block(+ json_schema={+ "type": "array",+ "items": {+ "type": "object",+ "properties": {+ "first_name": {"type": "string"},+ "last_name": {"type": "string"},+ },+ "required": ["first_name", "last_name"],+ },+ },+ )++ captured: dict[str, object] = {}++ async def fake_handler(*, prompt: str, prompt_name: str, force_dict: bool, **kwargs):+ captured["prompt"] = prompt+ captured["prompt_name"] = prompt_name+ captured["force_dict"] = force_dict+ return [{"first_name": "Zachary", "last_name": "Leibrand"}]++ async def fake_resolve_default_llm_handler(*args, **kwargs):+ return fake_handler++ def fake_get_override_handler(llm_key: str | None, *, default):+ return default++ monkeypatch.setattr(+ block_module.LLMAPIHandlerFactory,+ "get_override_llm_api_handler",+ fake_get_override_handler,+ raising=False,+ )+ monkeypatch.setattr(+ TextPromptBlock,+ "_resolve_default_llm_handler",+ fake_resolve_default_llm_handler,+ raising=False,+ )+ monkeypatch.setattr(+ prompt_engine,+ "load_prompt_from_string",+ lambda template, **kwargs: template,+ )++ response = await block.send_prompt(block.prompt, {}, workflow_run_id="workflow-run", organization_id="org-1")++ assert captured["prompt_name"] == "text-prompt"+ assert captured["force_dict"] is False+ assert response == [{"first_name": "Zachary", "last_name": "Leibrand"}]+++async def test_text_prompt_block_object_schema_does_not_force_dict_before_validation(monkeypatch):+ block = _make_text_prompt_block(+ json_schema={+ "type": "object",+ "properties": {+ "invoice_search_string": {"type": "string"},+ },+ "required": ["invoice_search_string"],+ },+ )++ captured: dict[str, object] = {}++ async def fake_handler(*, prompt: str, prompt_name: str, force_dict: bool, **kwargs):+ captured["force_dict"] = force_dict+ return {"invoice_search_string": "062026"}++ async def fake_resolve_default_llm_handler(*args, **kwargs):+ return fake_handler++ def fake_get_override_handler(llm_key: str | None, *, default):+ return default++ monkeypatch.setattr(+ block_module.LLMAPIHandlerFactory,+ "get_override_llm_api_handler",+ fake_get_override_handler,+ raising=False,+ )+ monkeypatch.setattr(+ TextPromptBlock,+ "_resolve_default_llm_handler",+ fake_resolve_default_llm_handler,+ raising=False,+ )+ monkeypatch.setattr(+ prompt_engine,+ "load_prompt_from_string",+ lambda template, **kwargs: template,+ )++ response = await block.send_prompt(block.prompt, {}, workflow_run_id="workflow-run", organization_id="org-1")++ assert captured["force_dict"] is False+ assert response == {"invoice_search_string": "062026"}+++async def test_text_prompt_block_retry_feedback_is_not_rendered_as_template(monkeypatch):+ block = _make_text_prompt_block(+ prompt="Hello {{ name }}",+ json_schema={+ "type": "object",+ "properties": {"answer": {"type": "string"}},+ "required": ["answer"],+ },+ )+ captured: dict[str, object] = {}+ rendered_templates: list[str] = []++ async def fake_handler(*, prompt: str, prompt_name: str, force_dict: bool, **kwargs):+ captured["prompt"] = prompt+ return {"answer": "ok"}++ async def fake_resolve_default_llm_handler(*args, **kwargs):+ return fake_handler++ def fake_get_override_handler(llm_key: str | None, *, default):+ return default++ def fake_load_prompt_from_string(template: str, **kwargs: str) -> str:+ rendered_templates.append(template)+ assert "previous response failed JSON schema validation" not in template+ return template.replace("{{ name }}", kwargs["name"])++ monkeypatch.setattr(+ block_module.LLMAPIHandlerFactory,+ "get_override_llm_api_handler",+ fake_get_override_handler,+ raising=False,+ )+ monkeypatch.setattr(+ TextPromptBlock,+ "_resolve_default_llm_handler",+ fake_resolve_default_llm_handler,+ raising=False,+ )+ monkeypatch.setattr(prompt_engine, "load_prompt_from_string", fake_load_prompt_from_string)++ await block.send_prompt(+ block.prompt,+ {"name": "Alice"},+ workflow_run_id="workflow-run",+ organization_id="org-1",+ schema_validation_failure="root: has 1 unexpected properties {{ dangerous_lookup }}",+ )++ sent_prompt = captured["prompt"]+ assert isinstance(sent_prompt, str)+ assert "Hello Alice" in sent_prompt+ assert "previous response failed JSON schema validation" in sent_prompt+ assert "{{ dangerous_lookup }}" in sent_prompt+++def test_text_prompt_block_schema_validation_rejects_wrong_root_type() -> None:+ block = _make_text_prompt_block(+ json_schema={+ "type": "array",+ "items": {+ "type": "object",+ "properties": {+ "first_name": {"type": "string"},+ "last_name": {"type": "string"},+ },+ "required": ["first_name", "last_name"],+ },+ },+ )++ failure_reason = block._validate_response_against_json_schema(+ {"first_name": "f_name_var", "last_name": "l_name_var\n```python\n..."}+ )++ assert failure_reason is not None+ assert "does not match text prompt JSON schema" in failure_reason+ assert "root" in failure_reason+ assert "expected type array, got object" in failure_reason+++def test_text_prompt_block_schema_validation_feedback_does_not_echo_invalid_response() -> None:+ secret_like_value = "customer-private-value-" + ("x" * 300)+ block = _make_text_prompt_block(+ json_schema={+ "type": "object",+ "properties": {+ "values": {+ "type": "array",+ "items": {"type": "string"},+ },+ },+ "required": ["values"],+ },+ )++ failure_reason = block._validate_response_against_json_schema({"values": secret_like_value})++ assert failure_reason is not None+ assert "root.values: expected type array, got string" in failure_reason+ assert secret_like_value not in failure_reason+ assert "customer-private-value" not in failure_reason+++def test_text_prompt_block_additional_properties_feedback_does_not_echo_response_keys() -> None:+ block = _make_text_prompt_block(+ json_schema={+ "type": "object",+ "properties": {"safe": {"type": "string"}},+ "additionalProperties": False,+ },+ )++ failure_reason = block._validate_response_against_json_schema({"safe": "ok", "[email protected]": "private"})++ assert failure_reason is not None+ assert "has 1 unexpected properties" in failure_reason+ assert "[email protected]" not in failure_reason+ assert "private" not in failure_reason+++def test_text_prompt_block_map_value_path_does_not_echo_response_key() -> None:+ block = _make_text_prompt_block(+ json_schema={+ "type": "object",+ "additionalProperties": {"type": "number"},+ },+ )++ failure_reason = block._validate_response_against_json_schema({"[email protected]": "not-a-number"})++ assert failure_reason is not None+ assert "root.<map value>: expected type number, got string" in failure_reason+ assert "[email protected]" not in failure_reason+ assert "not-a-number" not in failure_reason+++def test_text_prompt_block_unresolvable_ref_fails_schema_validation_gracefully() -> None:+ block = _make_text_prompt_block(json_schema={"$ref": "#/$defs/missing"})++ failure_reason = block._validate_response_against_json_schema({"value": "ok"})++ assert failure_reason is not None+ assert "Text prompt JSON schema validation failed" in failure_reason+++async def test_text_prompt_block_execute_fails_schema_validation_before_recording(monkeypatch):+ block = _make_text_prompt_block(+ json_schema={+ "type": "array",+ "items": {+ "type": "object",+ "properties": {+ "first_name": {"type": "string"},+ "last_name": {"type": "string"},+ },+ "required": ["first_name", "last_name"],+ },+ },+ )+ workflow_run_context = _make_workflow_run_context()+ bad_response = {"first_name": "f_name_var", "last_name": "code fragment"}+ record_output_parameter_value = AsyncMock()+ send_prompt = AsyncMock(return_value=bad_response)++ monkeypatch.setattr(TextPromptBlock, "send_prompt", send_prompt)+ monkeypatch.setattr(+ TextPromptBlock,+ "get_workflow_run_context",+ staticmethod(lambda workflow_run_id: workflow_run_context),+ )+ monkeypatch.setattr(TextPromptBlock, "record_output_parameter_value", record_output_parameter_value)++ result = await block.execute(+ workflow_run_id="workflow-run",+ workflow_run_block_id="workflow-run-block",+ organization_id="org-1",+ )++ assert result.success is False+ assert result.status == BlockStatus.failed+ assert result.output_parameter_value is None+ assert result.failure_reason is not None+ assert "does not match text prompt JSON schema" in result.failure_reason+ record_output_parameter_value.assert_not_awaited()+ assert send_prompt.await_count == block.schema_validation_max_attempts+++async def test_text_prompt_block_execute_invalid_schema_fails_without_llm_retry(monkeypatch):+ block = _make_text_prompt_block(json_schema={"type": 123})+ workflow_run_context = _make_workflow_run_context()+ record_output_parameter_value = AsyncMock()+ send_prompt = AsyncMock(return_value={"llm_response": "ok"})++ monkeypatch.setattr(TextPromptBlock, "send_prompt", send_prompt)+ monkeypatch.setattr(+ TextPromptBlock,+ "get_workflow_run_context",+ staticmethod(lambda workflow_run_id: workflow_run_context),+ )+ monkeypatch.setattr(TextPromptBlock, "record_output_parameter_value", record_output_parameter_value)++ result = await block.execute(+ workflow_run_id="workflow-run",+ workflow_run_block_id="workflow-run-block",+ organization_id="org-1",+ )++ assert result.success is False+ assert result.status == BlockStatus.failed+ assert result.failure_reason == "Text prompt JSON schema is invalid."+ send_prompt.assert_not_awaited()+ record_output_parameter_value.assert_not_awaited()+++async def test_text_prompt_block_execute_retries_schema_validation_failure(monkeypatch):+ block = _make_text_prompt_block(+ json_schema={+ "type": "array",+ "items": {+ "type": "object",+ "properties": {+ "first_name": {"type": "string"},+ "last_name": {"type": "string"},+ },+ "required": ["first_name", "last_name"],+ },+ },+ )+ workflow_run_context = _make_workflow_run_context()+ bad_response = {"first_name": "f_name_var", "last_name": "code fragment"}+ good_response = [{"first_name": "Zachary", "last_name": "Leibrand"}]+ responses = [bad_response, good_response]+ prompts: list[str] = []+ retry_failures: list[str | None] = []+ record_output_parameter_value = AsyncMock()++ async def fake_send_prompt(self, prompt, *args, **kwargs):+ prompts.append(prompt)+ retry_failures.append(kwargs.get("schema_validation_failure"))+ return responses.pop(0)++ monkeypatch.setattr(TextPromptBlock, "send_prompt", fake_send_prompt)+ monkeypatch.setattr(+ TextPromptBlock,+ "get_workflow_run_context",+ staticmethod(lambda workflow_run_id: workflow_run_context),+ )+ monkeypatch.setattr(TextPromptBlock, "record_output_parameter_value", record_output_parameter_value)++ result = await block.execute(+ workflow_run_id="workflow-run",+ workflow_run_block_id="workflow-run-block",+ organization_id="org-1",+ )++ assert result.success is True+ assert result.status == BlockStatus.completed+ assert result.output_parameter_value == good_response+ assert prompts == [block.prompt, block.prompt]+ assert retry_failures[0] is None+ assert retry_failures[1] is not None+ assert "expected type array, got object" in retry_failures[1]+ record_output_parameter_value.assert_awaited_once_with(workflow_run_context, "workflow-run", good_response)+++async def test_text_prompt_block_execute_retries_response_format_failure(monkeypatch):+ block = _make_text_prompt_block(+ json_schema={+ "type": "array",+ "items": {+ "type": "object",+ "properties": {+ "first_name": {"type": "string"},+ "last_name": {"type": "string"},+ },+ "required": ["first_name", "last_name"],+ },+ },+ )+ workflow_run_context = _make_workflow_run_context()+ raw_bad_response = "not-json-customer-private-value-" + ("x" * 300)+ good_response = [{"first_name": "Zachary", "last_name": "Leibrand"}]+ prompts: list[str] = []+ retry_failures: list[str | None] = []+ record_output_parameter_value = AsyncMock()++ async def fake_send_prompt(self, prompt, *args, **kwargs):+ prompts.append(prompt)+ retry_failures.append(kwargs.get("schema_validation_failure"))+ if len(prompts) == 1:+ raise InvalidLLMResponseFormat(raw_bad_response)+ return good_response++ monkeypatch.setattr(TextPromptBlock, "send_prompt", fake_send_prompt)+ monkeypatch.setattr(+ TextPromptBlock,+ "get_workflow_run_context",+ staticmethod(lambda workflow_run_id: workflow_run_context),+ )+ monkeypatch.setattr(TextPromptBlock, "record_output_parameter_value", record_output_parameter_value)++ result = await block.execute(+ workflow_run_id="workflow-run",+ workflow_run_block_id="workflow-run-block",+ organization_id="org-1",+ )++ assert result.success is True+ assert result.status == BlockStatus.completed+ assert result.output_parameter_value == good_response+ assert prompts == [block.prompt, block.prompt]+ assert retry_failures[0] is None+ assert retry_failures[1] is not None+ assert "InvalidLLMResponseFormat" in retry_failures[1]+ assert raw_bad_response not in retry_failures[1]+ assert "customer-private-value" not in retry_failures[1]+ record_output_parameter_value.assert_awaited_once_with(workflow_run_context, "workflow-run", good_response)++def test_workflow_request_deserialization_strips_invalid_text_prompt_llm_key() -> None:"""Verify FastAPI deserialization (not just explicit model_validate) strips bad keys.diff --git a/tests/unit/workflow/test_file_parser_block.py b/tests/unit/workflow/test_file_parser_block.pyindex ee36a7dad..23f64a9c4 100644--- a/tests/unit/workflow/test_file_parser_block.py+++ b/tests/unit/workflow/test_file_parser_block.py@@ -17,10 +17,11 @@ from unittest.mock import AsyncMock, MagicMockimport docximport pytest+from skyvern.forge.sdk.api.llm.exceptions import InvalidLLMResponseFormatfrom skyvern.forge.sdk.workflow.exceptions import InvalidFileType-from skyvern.forge.sdk.workflow.models.block import BlockType, FileParserBlock+from skyvern.forge.sdk.workflow.models.block import BlockType, FileParserBlock, PDFParserBlockfrom skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType-from skyvern.schemas.workflows import FileType+from skyvern.schemas.workflows import BlockResult, BlockStatus, FileTypedef _make_output_parameter(key: str) -> OutputParameter:@@ -45,6 +46,15 @@ def _make_file_parser_block(file_url: str, file_type: FileType) -> FileParserBlo)+def _make_pdf_parser_block(file_url: str) -> PDFParserBlock:+ return PDFParserBlock(+ label="test_pdf_parser",+ block_type=BlockType.PDF_PARSER,+ output_parameter=_make_output_parameter("test_output"),+ file_url=file_url,+ )++def _create_docx(path: Path,paragraphs: list[str] | None = None,@@ -384,6 +394,208 @@ class TestExtractWithAiSerialization:assert kwargs["extracted_text_content"] == "Hello\nWorld"+class TestExtractWithAiSchemaValidation:+ """Tests for schema adherence in FileParserBlock AI extraction."""++ @staticmethod+ def _patch_prompt_and_handler(mp: pytest.MonkeyPatch, handler: AsyncMock, prompt: str = "base prompt") -> None:+ mp.setattr(+ "skyvern.forge.sdk.workflow.models.block.LLMAPIHandlerFactory.get_override_llm_api_handler",+ lambda *a, **kw: handler,+ )+ mp.setattr(+ "skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt",+ MagicMock(return_value=prompt),+ )++ async def test_array_schema_does_not_force_dict(self) -> None:+ block = _make_file_parser_block("https://example.com/data.csv", FileType.CSV)+ block.json_schema = {+ "type": "array",+ "items": {+ "type": "object",+ "properties": {"name": {"type": "string"}},+ "required": ["name"],+ },+ }+ captured: dict[str, object] = {}++ async def fake_handler(**kwargs: Any) -> list[dict[str, str]]:+ captured["force_dict"] = kwargs["force_dict"]+ return [{"name": "Alice"}]++ with pytest.MonkeyPatch.context() as mp:+ handler = AsyncMock(side_effect=fake_handler)+ self._patch_prompt_and_handler(mp, handler)++ result = await block._extract_with_ai("name\nAlice", MagicMock())++ assert captured["force_dict"] is False+ assert result == [{"name": "Alice"}]++ async def test_object_schema_wrong_root_retries_without_prevalidation_coercion(self) -> None:+ block = _make_file_parser_block("https://example.com/data.csv", FileType.CSV)+ block.json_schema = {+ "type": "object",+ "properties": {"name": {"type": "string"}},+ "required": ["name"],+ }+ responses: list[Any] = [[{"name": "Alice"}], {"name": "Alice"}]+ prompts: list[str] = []+ force_dict_values: list[bool] = []++ async def fake_handler(**kwargs: Any) -> Any:+ prompts.append(kwargs["prompt"])+ force_dict_values.append(kwargs["force_dict"])+ return responses.pop(0)++ with pytest.MonkeyPatch.context() as mp:+ handler = AsyncMock(side_effect=fake_handler)+ self._patch_prompt_and_handler(mp, handler)++ result = await block._extract_with_ai("name\nAlice", MagicMock())++ assert result == {"name": "Alice"}+ assert handler.await_count == 2+ assert force_dict_values == [False, False]+ assert "previous response failed JSON schema validation" in prompts[1]+ assert "expected type object, got array" in prompts[1]++ async def test_invalid_schema_fails_before_llm_retry(self) -> None:+ block = _make_file_parser_block("https://example.com/data.csv", FileType.CSV)+ block.json_schema = {"type": 123}++ with pytest.MonkeyPatch.context() as mp:+ handler = AsyncMock()+ self._patch_prompt_and_handler(mp, handler)++ with pytest.raises(ValueError, match="File parser JSON schema is invalid"):+ await block._extract_with_ai("name\nAlice", MagicMock())++ handler.assert_not_awaited()++ async def test_schema_validation_failure_retries_with_sanitized_prompt(self) -> None:+ block = _make_file_parser_block("https://example.com/data.csv", FileType.CSV)+ block.json_schema = {+ "type": "array",+ "items": {+ "type": "object",+ "properties": {"name": {"type": "string"}},+ "required": ["name"],+ },+ }+ secret_like_value = "customer-private-value-" + ("x" * 300)+ responses: list[Any] = [{"name": secret_like_value}, [{"name": "Alice"}]]+ prompts: list[str] = []++ async def fake_handler(**kwargs: Any) -> Any:+ prompts.append(kwargs["prompt"])+ return responses.pop(0)++ with pytest.MonkeyPatch.context() as mp:+ handler = AsyncMock(side_effect=fake_handler)+ self._patch_prompt_and_handler(mp, handler)++ result = await block._extract_with_ai("name\nAlice", MagicMock())++ assert result == [{"name": "Alice"}]+ assert handler.await_count == 2+ assert prompts[0] == "base prompt"+ assert "previous response failed JSON schema validation" in prompts[1]+ assert "expected type array, got object" in prompts[1]+ assert secret_like_value not in prompts[1]+ assert "customer-private-value" not in prompts[1]++ async def test_response_format_failure_retries_with_sanitized_prompt(self) -> None:+ block = _make_file_parser_block("https://example.com/data.csv", FileType.CSV)+ block.json_schema = {+ "type": "object",+ "properties": {"name": {"type": "string"}},+ "required": ["name"],+ }+ raw_bad_response = "not-json-customer-private-value-" + ("x" * 300)+ prompts: list[str] = []++ async def fake_handler(**kwargs: Any) -> dict[str, str]:+ prompts.append(kwargs["prompt"])+ if len(prompts) == 1:+ raise InvalidLLMResponseFormat(raw_bad_response)+ return {"name": "Alice"}++ with pytest.MonkeyPatch.context() as mp:+ handler = AsyncMock(side_effect=fake_handler)+ self._patch_prompt_and_handler(mp, handler)++ result = await block._extract_with_ai("name\nAlice", MagicMock())++ assert result == {"name": "Alice"}+ assert handler.await_count == 2+ assert "InvalidLLMResponseFormat" in prompts[1]+ assert raw_bad_response not in prompts[1]+ assert "customer-private-value" not in prompts[1]+++class TestPDFParserSchemaValidation:+ """Tests for schema adherence in the deprecated PDFParserBlock."""++ @staticmethod+ def _patch_execute_dependencies(mp: pytest.MonkeyPatch, block: PDFParserBlock, handler: AsyncMock) -> AsyncMock:+ workflow_run_context = MagicMock()+ workflow_run_context.has_parameter.return_value = False+ record_output_parameter_value = AsyncMock()++ async def fake_build_block_result(self: PDFParserBlock, **kwargs: Any) -> BlockResult:+ kwargs.pop("organization_id", None)+ return BlockResult(output_parameter=self.output_parameter, **kwargs)++ mp.setattr(+ PDFParserBlock, "get_workflow_run_context", staticmethod(lambda workflow_run_id: workflow_run_context)+ )+ mp.setattr(PDFParserBlock, "record_output_parameter_value", record_output_parameter_value)+ mp.setattr(PDFParserBlock, "build_block_result", fake_build_block_result)+ mp.setattr("skyvern.forge.sdk.workflow.models.block.download_file", AsyncMock(return_value="/tmp/test.pdf"))+ mp.setattr("skyvern.forge.sdk.workflow.models.block.extract_pdf_file", MagicMock(return_value="name\nAlice"))+ mp.setattr(+ "skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt",+ MagicMock(return_value="base prompt"),+ )+ mp.setattr("skyvern.forge.sdk.workflow.models.block.app.LLM_API_HANDLER", handler)+ return record_output_parameter_value++ async def test_execute_retries_schema_validation_without_prevalidation_coercion(self) -> None:+ block = _make_pdf_parser_block("https://example.com/data.pdf")+ block.json_schema = {+ "type": "object",+ "properties": {"name": {"type": "string"}},+ "required": ["name"],+ }+ responses: list[Any] = [[{"name": "Alice"}], {"name": "Alice"}]+ prompts: list[str] = []+ force_dict_values: list[bool] = []++ async def fake_handler(**kwargs: Any) -> Any:+ prompts.append(kwargs["prompt"])+ force_dict_values.append(kwargs["force_dict"])+ return responses.pop(0)++ with pytest.MonkeyPatch.context() as mp:+ handler = AsyncMock(side_effect=fake_handler)+ record_output_parameter_value = self._patch_execute_dependencies(mp, block, handler)++ result = await block.execute("workflow-run", "workflow-run-block", organization_id="org-1")++ assert result.success is True+ assert result.status == BlockStatus.completed+ assert result.output_parameter_value == {"name": "Alice"}+ assert handler.await_count == 2+ assert force_dict_values == [False, False]+ assert "previous response failed JSON schema validation" in prompts[1]+ assert "expected type object, got array" in prompts[1]+ record_output_parameter_value.assert_awaited_once()++@pytest.mark.asyncioclass TestOcrPdfPages:"""Tests for the per-page vision-LLM OCR path for scanned PDFs (SKY-10960)."""@@ -427,6 +639,34 @@ class TestOcrPdfPages:assert "Cover sheet" in result and "Demographics" in resultassert result.index("--- Page 1 ---") < result.index("--- Page 2 ---") < result.index("--- Page 3 ---")+ async def test_invalid_ocr_response_retries_with_sanitized_prompt(self) -> None:+ """A missing extracted_text field is retried without echoing invalid content."""+ block = _make_file_parser_block("https://example.com/scan.pdf", FileType.PDF)+ secret_like_value = "customer-private-value-" + ("x" * 300)+ prompts: list[str] = []+ force_dict_values: list[bool] = []++ async def fake_handler(**kwargs: Any) -> dict[str, str]:+ prompts.append(kwargs["prompt"])+ force_dict_values.append(kwargs["force_dict"])+ if len(prompts) == 1:+ return {"wrong_field": secret_like_value}+ return {"extracted_text": "Recovered page text"}++ with pytest.MonkeyPatch.context() as mp:+ handler = AsyncMock(side_effect=fake_handler)+ self._patch_handler(mp, handler)++ result = await block._ocr_pdf_pages([b"img-1"])++ assert handler.await_count == 2+ assert force_dict_values == [False, False]+ assert "Recovered page text" in result+ assert "previous OCR response failed JSON validation" in prompts[1]+ assert "must include extracted_text as a string" in prompts[1]+ assert secret_like_value not in prompts[1]+ assert "customer-private-value" not in prompts[1]+async def test_order_preserved_when_a_later_page_resolves_first(self) -> None:"""Output stays in page order even if an earlier page's call finishes last."""block = _make_file_parser_block("https://example.com/scan.pdf", FileType.PDF)