Skip to content

Automate browser based workflows with AI

feat(SKY-11050): typed extraction schema for copilot code blocks (data_schema parity) (#6674)

Andrew Neilson64aa5cacdbb0
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/CodeBlockEditor.test.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/CodeBlockEditor.test.tsx
index 5321fa988..2ff0cb1a7 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/CodeBlockEditor.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/CodeBlockEditor.test.tsx
@@ -17,6 +17,7 @@ const baseData: CodeBlockNodeData = {
parameterKeys: [],
prompt: null,
steps: null,
+ dataSchema: "null",
model: null,
};
@@ -46,6 +47,15 @@ vi.mock("@/components/WorkflowBlockInputSet", () => ({
WorkflowBlockInputSet: () => null,
}));
+vi.mock(
+ "@/components/DataSchemaInputGroup/WorkflowDataSchemaInputGroup",
+ () => ({
+ WorkflowDataSchemaInputGroup: () => (
+ <div data-testid="data-schema-input-group" />
+ ),
+ }),
+);
+
vi.mock("@/components/WorkflowBlockInputTextarea", () => ({
WorkflowBlockInputTextarea: ({
value,
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/CodeBlockEditor.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/CodeBlockEditor.tsx
index 72a885314..5fe141d99 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/CodeBlockEditor.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/CodeBlockEditor.tsx
@@ -3,6 +3,7 @@ import { useReactFlow } from "@xyflow/react";
import { useMemo, useState } from "react";
import type { Extension } from "@uiw/react-codemirror";
+import { WorkflowDataSchemaInputGroup } from "@/components/DataSchemaInputGroup/WorkflowDataSchemaInputGroup";
import { Label } from "@/components/ui/label";
import { WorkflowBlockInputSet } from "@/components/WorkflowBlockInputSet";
import { WorkflowBlockInputTextarea } from "@/components/WorkflowBlockInputTextarea";
@@ -17,6 +18,7 @@ import { deepEqualStringArrays } from "@/util/equality";
import { cn } from "@/util/utils";
import { type AppNode, isWorkflowBlockNode } from "..";
+import { dataSchemaExampleValue } from "../types";
import { CodeBlockPlainCard } from "./CodeBlockPlainCard";
import type { CodeBlockView } from "./CodeBlockViewToggle";
import type { CodeBlockNode, CodeBlockNodeData } from "./types";
@@ -172,6 +174,18 @@ function CodeBlockEditorBody({
</div>
);
+ const dataSchemaField = (
+ <WorkflowDataSchemaInputGroup
+ value={data.dataSchema ?? "null"}
+ onChange={(value) => update({ dataSchema: value })}
+ exampleValue={dataSchemaExampleValue}
+ suggestionContext={{
+ data_extraction_goal: data.prompt ?? "",
+ current_schema: data.dataSchema ?? "null",
+ }}
+ />
+ );
+
if (isCodeFirst && view === "plain") {
return (
<div data-testid="code-block-block-form" className="space-y-4">
@@ -189,6 +203,7 @@ function CodeBlockEditorBody({
return (
<div data-testid="code-block-block-form" className="space-y-4">
{inputsField}
+ {dataSchemaField}
{codeEditorElement}
</div>
);
@@ -259,6 +274,7 @@ function CodeBlockEditorBody({
</div>
)}
{inputsField}
+ {isCodeFirst && dataSchemaField}
<div className="space-y-2">
<Label className="text-xs text-slate-300">Code Input</Label>
{codeEditorElement}
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/types.ts b/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/types.ts
index 41e309cb9..9d44ae14a 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/types.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/types.ts
@@ -10,6 +10,7 @@ export type CodeBlockNodeData = NodeBaseData & {
parameterKeys: Array<string> | null;
prompt: string | null;
steps: Array<CodeBlockStep> | null;
+ dataSchema: string;
};
export type CodeBlockNode = Node<CodeBlockNodeData, "codeBlock">;
@@ -34,5 +35,6 @@ export const codeBlockNodeDefaultData: CodeBlockNodeData = {
parameterKeys: null,
prompt: "",
steps: null,
+ dataSchema: "null",
model: null,
} as const;
diff --git a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts
index 255db9f3a..dda41a50c 100644
--- a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts
@@ -904,6 +904,7 @@ function convertToNode(
parameterKeys: block.parameters.map((p) => p.key),
prompt: block.prompt ?? null,
steps: block.steps ?? null,
+ dataSchema: "null",
},
};
}
diff --git a/skyvern/forge/sdk/copilot/code_block_synthesis.py b/skyvern/forge/sdk/copilot/code_block_synthesis.py
index ebcec8e35..88a1d771f 100644
--- a/skyvern/forge/sdk/copilot/code_block_synthesis.py
+++ b/skyvern/forge/sdk/copilot/code_block_synthesis.py
@@ -811,6 +811,10 @@ _FILL_CLAIM_ID = "claim:<fill>"
_FILL_CLAIM_TEXT = "<fill: the user-facing outcome this block claims>"
_FILL_CRITERION_ID = "criterion:<fill>"
_FILL_CRITERION_TEXT = "<fill: the terminal completion criterion>"
+_FILL_EXTRACTION_SCHEMA = (
+ "<fill: JSON Schema string of the fields to extract, e.g. "
+ '{"type":"object","properties":{"field_a":{"type":"string"}},"required":["field_a"]}>'
+)
def artifact_dependency_id(block_label: str) -> str:
@@ -868,6 +872,7 @@ def build_artifact_metadata_skeleton(
"depends_on": [dependency_id],
"covered_criteria": [_FILL_CRITERION_ID],
"goal_value_paths": ["<fill: output JSON path(s) carrying requested goal values>"],
+ "extraction_schema": _FILL_EXTRACTION_SCHEMA,
"observation_refs": [observation_ref_id],
}
],
@@ -981,9 +986,13 @@ def render_synthesized_offer_text(
parts.append(_render_artifact_metadata_block(metadata))
parts.append("```")
parts.append(
- "Shape the data-capture `return` so each `goal_value_paths` entry resolves to a named scalar field. "
- "For `records[].field` paths, return an array of objects; for a single record, return a keyed dict. "
- "Bind each `<fill: ...>` to the page text you captured. For example:"
+ "`extraction_schema` is the typed shape (named fields + types) of what this block extracts. "
+ "Propose it from the goal and the page text you scouted, surface the proposed fields to the user, "
+ "and ASK_QUESTION to confirm or adjust which fields to grab before committing the block. Carry the "
+ "confirmed JSON Schema back as `extraction_schema`; `goal_value_paths` index into it. Shape the "
+ "data-capture `return` so it conforms: each schema field is a named scalar key. For an array schema "
+ "return an array of objects; for a single record return a keyed dict. Bind each `<fill: ...>` to the "
+ "page text you captured. For example:"
)
parts.append("```python")
parts.append('return {"records": [{"field_a": "...", "field_b": "..."}]}')
diff --git a/skyvern/forge/sdk/copilot/tools/banned_blocks.py b/skyvern/forge/sdk/copilot/tools/banned_blocks.py
index b036b3b95..20985658b 100644
--- a/skyvern/forge/sdk/copilot/tools/banned_blocks.py
+++ b/skyvern/forge/sdk/copilot/tools/banned_blocks.py
@@ -256,6 +256,10 @@ Runtime facts:
`fill_credential_field`, never embed literal secrets.
- Return JSON-safe structured data plus visible evidence text for records, totals,
confirmations, and identifiers.
+- For an extraction-intent `code` block, propose a typed `extraction_schema` (named
+ fields with types) from the goal and the scouted page, ASK_QUESTION to confirm or
+ adjust which fields to grab, then carry the confirmed JSON Schema as
+ `extraction_schema` on `code_artifact_metadata` and conform the block's `return` to it.
- Use YAML block scalars (`code: |`) and pass complete workflow YAML to update tools.
""".strip()
diff --git a/skyvern/forge/sdk/copilot/tools/blockers.py b/skyvern/forge/sdk/copilot/tools/blockers.py
index 1cfe825bc..21a36d6b5 100644
--- a/skyvern/forge/sdk/copilot/tools/blockers.py
+++ b/skyvern/forge/sdk/copilot/tools/blockers.py
@@ -9,6 +9,7 @@ import structlog
from skyvern.forge import app
from skyvern.forge.failure_classifier import classify_from_failure_reason
+from skyvern.forge.sdk.api.llm.schema_validator import validate_and_fill_extraction_result
from skyvern.forge.sdk.copilot.blocker_signal import (
CopilotToolBlockerSignal,
assert_clean_user_facing_text,
@@ -1101,6 +1102,42 @@ def _goal_value_paths_for_code_block(copilot_ctx: Any | None, label: Any) -> lis
return paths
+def _parse_metadata_extraction_schema(value: Any) -> dict[str, Any] | None:
+ # Keep in sync with workflow_update._parse_extraction_schema; duplicated locally
+ # so runtime blockers do not import the authoring validator.
+ if isinstance(value, dict):
+ return value or None
+ if not isinstance(value, str):
+ return None
+ text = value.strip()
+ if not text or text.casefold() in {"null", "none"} or text.casefold().startswith("<fill"):
+ return None
+ try:
+ parsed = json.loads(text)
+ except (json.JSONDecodeError, ValueError):
+ return None
+ return parsed if isinstance(parsed, dict) and parsed else None
+
+
+def _extraction_schema_for_code_block(copilot_ctx: Any | None, label: Any) -> dict[str, Any] | None:
+ if copilot_ctx is None or not isinstance(label, str):
+ return None
+ metadata = getattr(copilot_ctx, "code_artifact_metadata", None)
+ if not isinstance(metadata, dict):
+ return None
+ entry = metadata.get(label)
+ if not isinstance(entry, dict):
+ return None
+ # claimed_outcomes wins when both groups declare a schema; they are expected to carry the same value.
+ for row_group in (entry.get("claimed_outcomes"), entry.get("terminal_verifier_expectations")):
+ rows = [row for row in row_group if isinstance(row, dict)] if isinstance(row_group, list) else []
+ for row in rows:
+ schema = _parse_metadata_extraction_schema(row.get("extraction_schema"))
+ if schema is not None:
+ return schema
+ return None
+
+
_GOAL_PATH_INDEX_PATTERN = re.compile(r"\[\d+\]")
@@ -1599,6 +1636,15 @@ def _analyze_run_blocks(
)
if structured_blocker:
texts_to_scan.append(structured_blocker)
+ extraction_schema = _extraction_schema_for_code_block(copilot_ctx, block.get("label"))
+ # Array-typed schemas would coerce the keyed dict return to [] (fill_missing_fields), making the
+ # goal-path check below read a real extraction as empty; the keyed-return floor guarantees a dict.
+ if (
+ extraction_schema is not None
+ and isinstance(extracted, dict)
+ and extraction_schema.get("type") != "array"
+ ):
+ extracted = validate_and_fill_extraction_result(extracted, extraction_schema)
goal_value_paths = _goal_value_paths_for_code_block(copilot_ctx, block.get("label"))
if goal_value_paths:
has_data_blocks = True
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index ebed9ed00..a6c9ee74f 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import ast
+import json
import re
import textwrap
from collections.abc import Iterator, Mapping
@@ -13,6 +14,7 @@ import yaml
from pydantic import AliasChoices, BaseModel, Field, ValidationError
from skyvern.forge import app
+from skyvern.forge.sdk.api.llm.schema_validator import validate_schema
from skyvern.forge.sdk.copilot.attribution import resolve_copilot_created_by_stamp
from skyvern.forge.sdk.copilot.blocker_signal import clear_terminal_evidence_on_workflow_edit
from skyvern.forge.sdk.copilot.code_block_preflight import sandbox_unresolved_name_diagnostics
@@ -114,6 +116,15 @@ class CodeArtifactClaimedOutcome(BaseModel):
"`records[].number` or `records[].expiration_date`."
),
)
+ extraction_schema: str | None = Field(
+ default=None,
+ description=(
+ "JSON Schema the user confirmed for this claim's extraction shape, serialized as a JSON "
+ 'string (an object, or `{"type":"array","items":{...}}` for repeated records). Named '
+ "fields, types, and nesting the `goal_value_paths` index into; the block return is conformed "
+ "and validated against it. Same dialect as the legacy `data_schema` lever."
+ ),
+ )
evidence_refs: list[str] = Field(default_factory=list)
observation_refs: list[str] = Field(default_factory=list)
required_tokens: list[str] = Field(default_factory=list)
@@ -166,6 +177,10 @@ class CodeArtifactTerminalVerifierExpectation(BaseModel):
default_factory=list,
description="Output JSON paths terminal verification should treat as goal-value evidence.",
)
+ extraction_schema: str | None = Field(
+ default=None,
+ description="JSON Schema (serialized JSON string) of the confirmed extraction shape terminal verification expects.",
+ )
class CodeArtifactExplorationObservation(BaseModel):
@@ -401,6 +416,9 @@ def _normalize_code_artifact_metadata_detailed(
return_shape_error = _extraction_return_shape_error(label, dumped, block_code)
if return_shape_error is not None:
item_violations.append(return_shape_error)
+ schema_conformance_error = _extraction_schema_conformance_error(label, dumped, block_code)
+ if schema_conformance_error is not None:
+ item_violations.append(schema_conformance_error)
download_shape_error = _download_return_shape_error(label, dumped, block_code)
if download_shape_error is not None:
item_violations.append(download_shape_error)
@@ -1559,6 +1577,133 @@ def _extraction_return_shape_error(label: str, artifact: Mapping[str, Any], code
return None
+def _parse_extraction_schema(value: Any) -> dict[str, Any] | None:
+ """Coerce a declared `extraction_schema` to a JSON-Schema object dict, or None when
+ absent / disabled (`null`, empty) / still an unfilled `<fill...>` placeholder / not a
+ parseable object. Accepts a JSON string (the tool-facing form) or an already-decoded
+ dict (FE / direct callers)."""
+ if isinstance(value, dict):
+ return value or None
+ if not isinstance(value, str):
+ return None
+ text = value.strip()
+ if not text or text.casefold() in {"null", "none"} or text.casefold().startswith("<fill"):
+ return None
+ try:
+ parsed = json.loads(text)
+ except (json.JSONDecodeError, ValueError):
+ return None
+ return parsed if isinstance(parsed, dict) and parsed else None
+
+
+def _artifact_extraction_schema_values(artifact: Mapping[str, Any]) -> list[Any]:
+ values: list[Any] = []
+ for field_name in ("claimed_outcomes", "terminal_verifier_expectations"):
+ for row in _artifact_rows(artifact.get(field_name)):
+ schema = row.get("extraction_schema")
+ if schema is not None and not (isinstance(schema, str) and not schema.strip()):
+ values.append(schema)
+ return values
+
+
+def _schema_object_property_names(schema: Mapping[str, Any]) -> tuple[set[str], set[str]] | None:
+ """Property names and required names for the record-level object of a data schema.
+
+ Returns the (properties, required) name sets for an `object` schema, or for the
+ `items` object of an `array` schema (records-style). Returns None when the schema
+ declares no statically-readable record object, so reconciliation degrades to tolerant."""
+ schema_type = schema.get("type")
+ if schema_type == "array":
+ items = schema.get("items")
+ if isinstance(items, Mapping):
+ return _schema_object_property_names(items)
+ return None
+ properties = schema.get("properties")
+ if not isinstance(properties, Mapping):
+ return None
+ property_names = {str(name) for name in properties}
+ required = schema.get("required")
+ required_names = {str(name) for name in required} if isinstance(required, list) else set()
+ return property_names, required_names & property_names
+
+
+def _top_level_return_dict_keys(code: str) -> set[str] | None:
+ """Top-level string keys of the snippet's returned dict literal (or the record
+ objects inside a returned list literal). Returns None when no top-level dict/list
+ literal return is statically determinable, so a dynamically-built return is never
+ false-rejected and falls through to the runtime validate/fill pass."""
+ try:
+ tree = ast.parse(textwrap.dedent(code).strip() or "pass")
+ except SyntaxError:
+ return None
+
+ return_values: list[ast.expr] = []
+ for node in _iter_top_level_scope(tree.body):
+ if isinstance(node, ast.Return) and node.value is not None:
+ return_values.append(node.value)
+ if not return_values:
+ return None
+
+ keys: set[str] = set()
+ for value in return_values:
+ unwrapped = value.value if isinstance(value, ast.Await) else value
+ if isinstance(unwrapped, ast.Dict):
+ keys |= _dict_keys(unwrapped)
+ elif isinstance(unwrapped, ast.List):
+ for element in unwrapped.elts:
+ if isinstance(element, ast.Dict):
+ keys |= _dict_keys(element)
+ else:
+ return None
+ else:
+ return None
+ return keys
+
+
+def _extraction_schema_conformance_error(label: str, artifact: Mapping[str, Any], code: str) -> str | None:
+ """Enforce the confirmed `extraction_schema` against the authored return shape. Runs
+ in addition to (never instead of) `_extraction_return_shape_error`. A declared-but-malformed
+ schema is rejected; a top-level return dict literal whose keys omit a required schema field is
+ rejected; a dynamically-built return cannot be statically reconciled and defers to the
+ runtime validate/fill pass."""
+ declared = _artifact_extraction_schema_values(artifact)
+ schemas: list[dict[str, Any]] = []
+ for value in declared:
+ # An unfilled `<fill...>` slot is a not-yet-confirmed schema, not a malformed one;
+ # leave it for the model to fill, the same way unfilled goal_value_paths are tolerated.
+ if isinstance(value, str) and _is_unfilled_artifact_placeholder(value.strip()):
+ continue
+ parsed = _parse_extraction_schema(value)
+ if parsed is None or not validate_schema(parsed):
+ return (
+ f"Code block `{label}` declares an `extraction_schema` that is not valid JSON Schema. "
+ "Provide a JSON Schema (a JSON object with named fields and types, serialized as a string), or "
+ "remove `extraction_schema` to fall back to `goal_value_paths` alone."
+ )
+ schemas.append(parsed)
+ if not schemas:
+ return None
+ if _is_download_intent(artifact, code) or not code.strip():
+ return None
+ return_keys = _top_level_return_dict_keys(code)
+ if return_keys is None:
+ return None
+ for schema in schemas:
+ names = _schema_object_property_names(schema)
+ if names is None:
+ continue
+ _property_names, required_names = names
+ missing_required = sorted(required_names - return_keys)
+ if missing_required:
+ return (
+ f"Code block `{label}` `return`s a keyed structure missing required field(s) "
+ f"{', '.join(missing_required)} from the confirmed `extraction_schema`. Build the top-level "
+ "return so every required schema field is a named key (a dict for one record, or an array of "
+ "objects with those keys for repeated records)."
+ )
+ return None
+
+
_EXPECT_DOWNLOAD_ATTR = "expect_download"