Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(SKY-11050): typed extraction schema for copilot code blocks (data_schema parity) (#6674)
Andrew Neilson64aa5cacdbb0
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/CodeBlockEditor.test.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/CodeBlockEditor.test.tsxindex 5321fa988..2ff0cb1a7 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/CodeBlockEditor.test.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/CodeBlockEditor.test.tsx@@ -17,6 +17,7 @@ const baseData: CodeBlockNodeData = {parameterKeys: [],prompt: null,steps: null,+ dataSchema: "null",model: null,};@@ -46,6 +47,15 @@ vi.mock("@/components/WorkflowBlockInputSet", () => ({WorkflowBlockInputSet: () => null,}));+vi.mock(+ "@/components/DataSchemaInputGroup/WorkflowDataSchemaInputGroup",+ () => ({+ WorkflowDataSchemaInputGroup: () => (+ <div data-testid="data-schema-input-group" />+ ),+ }),+);+vi.mock("@/components/WorkflowBlockInputTextarea", () => ({WorkflowBlockInputTextarea: ({value,diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/CodeBlockEditor.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/CodeBlockEditor.tsxindex 72a885314..5fe141d99 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/CodeBlockEditor.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/CodeBlockEditor.tsx@@ -3,6 +3,7 @@ import { useReactFlow } from "@xyflow/react";import { useMemo, useState } from "react";import type { Extension } from "@uiw/react-codemirror";+import { WorkflowDataSchemaInputGroup } from "@/components/DataSchemaInputGroup/WorkflowDataSchemaInputGroup";import { Label } from "@/components/ui/label";import { WorkflowBlockInputSet } from "@/components/WorkflowBlockInputSet";import { WorkflowBlockInputTextarea } from "@/components/WorkflowBlockInputTextarea";@@ -17,6 +18,7 @@ import { deepEqualStringArrays } from "@/util/equality";import { cn } from "@/util/utils";import { type AppNode, isWorkflowBlockNode } from "..";+import { dataSchemaExampleValue } from "../types";import { CodeBlockPlainCard } from "./CodeBlockPlainCard";import type { CodeBlockView } from "./CodeBlockViewToggle";import type { CodeBlockNode, CodeBlockNodeData } from "./types";@@ -172,6 +174,18 @@ function CodeBlockEditorBody({</div>);+ const dataSchemaField = (+ <WorkflowDataSchemaInputGroup+ value={data.dataSchema ?? "null"}+ onChange={(value) => update({ dataSchema: value })}+ exampleValue={dataSchemaExampleValue}+ suggestionContext={{+ data_extraction_goal: data.prompt ?? "",+ current_schema: data.dataSchema ?? "null",+ }}+ />+ );+if (isCodeFirst && view === "plain") {return (<div data-testid="code-block-block-form" className="space-y-4">@@ -189,6 +203,7 @@ function CodeBlockEditorBody({return (<div data-testid="code-block-block-form" className="space-y-4">{inputsField}+ {dataSchemaField}{codeEditorElement}</div>);@@ -259,6 +274,7 @@ function CodeBlockEditorBody({</div>)}{inputsField}+ {isCodeFirst && dataSchemaField}<div className="space-y-2"><Label className="text-xs text-slate-300">Code Input</Label>{codeEditorElement}diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/types.ts b/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/types.tsindex 41e309cb9..9d44ae14a 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/types.ts+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/types.ts@@ -10,6 +10,7 @@ export type CodeBlockNodeData = NodeBaseData & {parameterKeys: Array<string> | null;prompt: string | null;steps: Array<CodeBlockStep> | null;+ dataSchema: string;};export type CodeBlockNode = Node<CodeBlockNodeData, "codeBlock">;@@ -34,5 +35,6 @@ export const codeBlockNodeDefaultData: CodeBlockNodeData = {parameterKeys: null,prompt: "",steps: null,+ dataSchema: "null",model: null,} as const;diff --git a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.tsindex 255db9f3a..dda41a50c 100644--- a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts+++ b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts@@ -904,6 +904,7 @@ function convertToNode(parameterKeys: block.parameters.map((p) => p.key),prompt: block.prompt ?? null,steps: block.steps ?? null,+ dataSchema: "null",},};}diff --git a/skyvern/forge/sdk/copilot/code_block_synthesis.py b/skyvern/forge/sdk/copilot/code_block_synthesis.pyindex ebcec8e35..88a1d771f 100644--- a/skyvern/forge/sdk/copilot/code_block_synthesis.py+++ b/skyvern/forge/sdk/copilot/code_block_synthesis.py@@ -811,6 +811,10 @@ _FILL_CLAIM_ID = "claim:<fill>"_FILL_CLAIM_TEXT = "<fill: the user-facing outcome this block claims>"_FILL_CRITERION_ID = "criterion:<fill>"_FILL_CRITERION_TEXT = "<fill: the terminal completion criterion>"+_FILL_EXTRACTION_SCHEMA = (+ "<fill: JSON Schema string of the fields to extract, e.g. "+ '{"type":"object","properties":{"field_a":{"type":"string"}},"required":["field_a"]}>'+)def artifact_dependency_id(block_label: str) -> str:@@ -868,6 +872,7 @@ def build_artifact_metadata_skeleton("depends_on": [dependency_id],"covered_criteria": [_FILL_CRITERION_ID],"goal_value_paths": ["<fill: output JSON path(s) carrying requested goal values>"],+ "extraction_schema": _FILL_EXTRACTION_SCHEMA,"observation_refs": [observation_ref_id],}],@@ -981,9 +986,13 @@ def render_synthesized_offer_text(parts.append(_render_artifact_metadata_block(metadata))parts.append("```")parts.append(- "Shape the data-capture `return` so each `goal_value_paths` entry resolves to a named scalar field. "- "For `records[].field` paths, return an array of objects; for a single record, return a keyed dict. "- "Bind each `<fill: ...>` to the page text you captured. For example:"+ "`extraction_schema` is the typed shape (named fields + types) of what this block extracts. "+ "Propose it from the goal and the page text you scouted, surface the proposed fields to the user, "+ "and ASK_QUESTION to confirm or adjust which fields to grab before committing the block. Carry the "+ "confirmed JSON Schema back as `extraction_schema`; `goal_value_paths` index into it. Shape the "+ "data-capture `return` so it conforms: each schema field is a named scalar key. For an array schema "+ "return an array of objects; for a single record return a keyed dict. Bind each `<fill: ...>` to the "+ "page text you captured. For example:")parts.append("```python")parts.append('return {"records": [{"field_a": "...", "field_b": "..."}]}')diff --git a/skyvern/forge/sdk/copilot/tools/banned_blocks.py b/skyvern/forge/sdk/copilot/tools/banned_blocks.pyindex b036b3b95..20985658b 100644--- a/skyvern/forge/sdk/copilot/tools/banned_blocks.py+++ b/skyvern/forge/sdk/copilot/tools/banned_blocks.py@@ -256,6 +256,10 @@ Runtime facts:`fill_credential_field`, never embed literal secrets.- Return JSON-safe structured data plus visible evidence text for records, totals,confirmations, and identifiers.+- For an extraction-intent `code` block, propose a typed `extraction_schema` (named+ fields with types) from the goal and the scouted page, ASK_QUESTION to confirm or+ adjust which fields to grab, then carry the confirmed JSON Schema as+ `extraction_schema` on `code_artifact_metadata` and conform the block's `return` to it.- Use YAML block scalars (`code: |`) and pass complete workflow YAML to update tools.""".strip()diff --git a/skyvern/forge/sdk/copilot/tools/blockers.py b/skyvern/forge/sdk/copilot/tools/blockers.pyindex 1cfe825bc..21a36d6b5 100644--- a/skyvern/forge/sdk/copilot/tools/blockers.py+++ b/skyvern/forge/sdk/copilot/tools/blockers.py@@ -9,6 +9,7 @@ import structlogfrom skyvern.forge import appfrom skyvern.forge.failure_classifier import classify_from_failure_reason+from skyvern.forge.sdk.api.llm.schema_validator import validate_and_fill_extraction_resultfrom skyvern.forge.sdk.copilot.blocker_signal import (CopilotToolBlockerSignal,assert_clean_user_facing_text,@@ -1101,6 +1102,42 @@ def _goal_value_paths_for_code_block(copilot_ctx: Any | None, label: Any) -> lisreturn paths+def _parse_metadata_extraction_schema(value: Any) -> dict[str, Any] | None:+ # Keep in sync with workflow_update._parse_extraction_schema; duplicated locally+ # so runtime blockers do not import the authoring validator.+ if isinstance(value, dict):+ return value or None+ if not isinstance(value, str):+ return None+ text = value.strip()+ if not text or text.casefold() in {"null", "none"} or text.casefold().startswith("<fill"):+ return None+ try:+ parsed = json.loads(text)+ except (json.JSONDecodeError, ValueError):+ return None+ return parsed if isinstance(parsed, dict) and parsed else None+++def _extraction_schema_for_code_block(copilot_ctx: Any | None, label: Any) -> dict[str, Any] | None:+ if copilot_ctx is None or not isinstance(label, str):+ return None+ metadata = getattr(copilot_ctx, "code_artifact_metadata", None)+ if not isinstance(metadata, dict):+ return None+ entry = metadata.get(label)+ if not isinstance(entry, dict):+ return None+ # claimed_outcomes wins when both groups declare a schema; they are expected to carry the same value.+ for row_group in (entry.get("claimed_outcomes"), entry.get("terminal_verifier_expectations")):+ rows = [row for row in row_group if isinstance(row, dict)] if isinstance(row_group, list) else []+ for row in rows:+ schema = _parse_metadata_extraction_schema(row.get("extraction_schema"))+ if schema is not None:+ return schema+ return None++_GOAL_PATH_INDEX_PATTERN = re.compile(r"\[\d+\]")@@ -1599,6 +1636,15 @@ def _analyze_run_blocks()if structured_blocker:texts_to_scan.append(structured_blocker)+ extraction_schema = _extraction_schema_for_code_block(copilot_ctx, block.get("label"))+ # Array-typed schemas would coerce the keyed dict return to [] (fill_missing_fields), making the+ # goal-path check below read a real extraction as empty; the keyed-return floor guarantees a dict.+ if (+ extraction_schema is not None+ and isinstance(extracted, dict)+ and extraction_schema.get("type") != "array"+ ):+ extracted = validate_and_fill_extraction_result(extracted, extraction_schema)goal_value_paths = _goal_value_paths_for_code_block(copilot_ctx, block.get("label"))if goal_value_paths:has_data_blocks = Truediff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex ebed9ed00..a6c9ee74f 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -1,6 +1,7 @@from __future__ import annotationsimport ast+import jsonimport reimport textwrapfrom collections.abc import Iterator, Mapping@@ -13,6 +14,7 @@ import yamlfrom pydantic import AliasChoices, BaseModel, Field, ValidationErrorfrom skyvern.forge import app+from skyvern.forge.sdk.api.llm.schema_validator import validate_schemafrom skyvern.forge.sdk.copilot.attribution import resolve_copilot_created_by_stampfrom skyvern.forge.sdk.copilot.blocker_signal import clear_terminal_evidence_on_workflow_editfrom skyvern.forge.sdk.copilot.code_block_preflight import sandbox_unresolved_name_diagnostics@@ -114,6 +116,15 @@ class CodeArtifactClaimedOutcome(BaseModel):"`records[].number` or `records[].expiration_date`."),)+ extraction_schema: str | None = Field(+ default=None,+ description=(+ "JSON Schema the user confirmed for this claim's extraction shape, serialized as a JSON "+ 'string (an object, or `{"type":"array","items":{...}}` for repeated records). Named '+ "fields, types, and nesting the `goal_value_paths` index into; the block return is conformed "+ "and validated against it. Same dialect as the legacy `data_schema` lever."+ ),+ )evidence_refs: list[str] = Field(default_factory=list)observation_refs: list[str] = Field(default_factory=list)required_tokens: list[str] = Field(default_factory=list)@@ -166,6 +177,10 @@ class CodeArtifactTerminalVerifierExpectation(BaseModel):default_factory=list,description="Output JSON paths terminal verification should treat as goal-value evidence.",)+ extraction_schema: str | None = Field(+ default=None,+ description="JSON Schema (serialized JSON string) of the confirmed extraction shape terminal verification expects.",+ )class CodeArtifactExplorationObservation(BaseModel):@@ -401,6 +416,9 @@ def _normalize_code_artifact_metadata_detailed(return_shape_error = _extraction_return_shape_error(label, dumped, block_code)if return_shape_error is not None:item_violations.append(return_shape_error)+ schema_conformance_error = _extraction_schema_conformance_error(label, dumped, block_code)+ if schema_conformance_error is not None:+ item_violations.append(schema_conformance_error)download_shape_error = _download_return_shape_error(label, dumped, block_code)if download_shape_error is not None:item_violations.append(download_shape_error)@@ -1559,6 +1577,133 @@ def _extraction_return_shape_error(label: str, artifact: Mapping[str, Any], codereturn None+def _parse_extraction_schema(value: Any) -> dict[str, Any] | None:+ """Coerce a declared `extraction_schema` to a JSON-Schema object dict, or None when+ absent / disabled (`null`, empty) / still an unfilled `<fill...>` placeholder / not a+ parseable object. Accepts a JSON string (the tool-facing form) or an already-decoded+ dict (FE / direct callers)."""+ if isinstance(value, dict):+ return value or None+ if not isinstance(value, str):+ return None+ text = value.strip()+ if not text or text.casefold() in {"null", "none"} or text.casefold().startswith("<fill"):+ return None+ try:+ parsed = json.loads(text)+ except (json.JSONDecodeError, ValueError):+ return None+ return parsed if isinstance(parsed, dict) and parsed else None+++def _artifact_extraction_schema_values(artifact: Mapping[str, Any]) -> list[Any]:+ values: list[Any] = []+ for field_name in ("claimed_outcomes", "terminal_verifier_expectations"):+ for row in _artifact_rows(artifact.get(field_name)):+ schema = row.get("extraction_schema")+ if schema is not None and not (isinstance(schema, str) and not schema.strip()):+ values.append(schema)+ return values+++def _schema_object_property_names(schema: Mapping[str, Any]) -> tuple[set[str], set[str]] | None:+ """Property names and required names for the record-level object of a data schema.++ Returns the (properties, required) name sets for an `object` schema, or for the+ `items` object of an `array` schema (records-style). Returns None when the schema+ declares no statically-readable record object, so reconciliation degrades to tolerant."""+ schema_type = schema.get("type")+ if schema_type == "array":+ items = schema.get("items")+ if isinstance(items, Mapping):+ return _schema_object_property_names(items)+ return None+ properties = schema.get("properties")+ if not isinstance(properties, Mapping):+ return None+ property_names = {str(name) for name in properties}+ required = schema.get("required")+ required_names = {str(name) for name in required} if isinstance(required, list) else set()+ return property_names, required_names & property_names+++def _top_level_return_dict_keys(code: str) -> set[str] | None:+ """Top-level string keys of the snippet's returned dict literal (or the record+ objects inside a returned list literal). Returns None when no top-level dict/list+ literal return is statically determinable, so a dynamically-built return is never+ false-rejected and falls through to the runtime validate/fill pass."""+ try:+ tree = ast.parse(textwrap.dedent(code).strip() or "pass")+ except SyntaxError:+ return None++ return_values: list[ast.expr] = []+ for node in _iter_top_level_scope(tree.body):+ if isinstance(node, ast.Return) and node.value is not None:+ return_values.append(node.value)+ if not return_values:+ return None++ keys: set[str] = set()+ for value in return_values:+ unwrapped = value.value if isinstance(value, ast.Await) else value+ if isinstance(unwrapped, ast.Dict):+ keys |= _dict_keys(unwrapped)+ elif isinstance(unwrapped, ast.List):+ for element in unwrapped.elts:+ if isinstance(element, ast.Dict):+ keys |= _dict_keys(element)+ else:+ return None+ else:+ return None+ return keys+++def _extraction_schema_conformance_error(label: str, artifact: Mapping[str, Any], code: str) -> str | None:+ """Enforce the confirmed `extraction_schema` against the authored return shape. Runs+ in addition to (never instead of) `_extraction_return_shape_error`. A declared-but-malformed+ schema is rejected; a top-level return dict literal whose keys omit a required schema field is+ rejected; a dynamically-built return cannot be statically reconciled and defers to the+ runtime validate/fill pass."""+ declared = _artifact_extraction_schema_values(artifact)+ schemas: list[dict[str, Any]] = []+ for value in declared:+ # An unfilled `<fill...>` slot is a not-yet-confirmed schema, not a malformed one;+ # leave it for the model to fill, the same way unfilled goal_value_paths are tolerated.+ if isinstance(value, str) and _is_unfilled_artifact_placeholder(value.strip()):+ continue+ parsed = _parse_extraction_schema(value)+ if parsed is None or not validate_schema(parsed):+ return (+ f"Code block `{label}` declares an `extraction_schema` that is not valid JSON Schema. "+ "Provide a JSON Schema (a JSON object with named fields and types, serialized as a string), or "+ "remove `extraction_schema` to fall back to `goal_value_paths` alone."+ )+ schemas.append(parsed)+ if not schemas:+ return None+ if _is_download_intent(artifact, code) or not code.strip():+ return None+ return_keys = _top_level_return_dict_keys(code)+ if return_keys is None:+ return None+ for schema in schemas:+ names = _schema_object_property_names(schema)+ if names is None:+ continue+ _property_names, required_names = names+ missing_required = sorted(required_names - return_keys)+ if missing_required:+ return (+ f"Code block `{label}` `return`s a keyed structure missing required field(s) "+ f"{', '.join(missing_required)} from the confirmed `extraction_schema`. Build the top-level "+ "return so every required schema field is a named key (a dict for one record, or an array of "+ "objects with those keys for repeated records)."+ )+ return None++_EXPECT_DOWNLOAD_ATTR = "expect_download"