Skip to content

Automate browser based workflows with AI

feat(workflow): expose data_extraction_goal and data_schema on ValidationBlock backend (#6066)

Aaron Perez6ccca74c3ea7
diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.json
index 8b6847625..22e2ada20 100644
--- a/docs/api-reference/openapi.json
+++ b/docs/api-reference/openapi.json
@@ -12059,6 +12059,11 @@
"title": "Next Loop On Failure",
"default": false
},
+ "navigation_goal": {
+ "type": "string",
+ "nullable": true,
+ "title": "Navigation Goal"
+ },
"complete_criterion": {
"type": "string",
"nullable": true,
@@ -12089,6 +12094,28 @@
"type": "boolean",
"title": "Disable Cache",
"default": false
+ },
+ "data_extraction_goal": {
+ "type": "string",
+ "nullable": true,
+ "title": "Data Extraction Goal"
+ },
+ "data_schema": {
+ "anyOf": [
+ {
+ "additionalProperties": true,
+ "type": "object"
+ },
+ {
+ "items": {},
+ "type": "array"
+ },
+ {
+ "type": "string"
+ }
+ ],
+ "title": "Data Schema",
+ "nullable": true
}
},
"type": "object",
diff --git a/fern/openapi/skyvern_openapi.json b/fern/openapi/skyvern_openapi.json
index 7e4193fed..8fe316071 100644
--- a/fern/openapi/skyvern_openapi.json
+++ b/fern/openapi/skyvern_openapi.json
@@ -19164,6 +19164,17 @@
"title": "Next Loop On Failure",
"default": false
},
+ "navigation_goal": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Navigation Goal"
+ },
"complete_criterion": {
"anyOf": [
{
@@ -19218,6 +19229,36 @@
"type": "boolean",
"title": "Disable Cache",
"default": false
+ },
+ "data_extraction_goal": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Data Extraction Goal"
+ },
+ "data_schema": {
+ "anyOf": [
+ {
+ "additionalProperties": true,
+ "type": "object"
+ },
+ {
+ "items": {},
+ "type": "array"
+ },
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Data Schema"
}
},
"type": "object",
diff --git a/skyvern/client/types/__init__.py b/skyvern/client/types/__init__.py
index 38799a4df..172687f95 100644
--- a/skyvern/client/types/__init__.py
+++ b/skyvern/client/types/__init__.py
@@ -459,6 +459,7 @@ if typing.TYPE_CHECKING:
ValidationBlockParametersItem_Workflow,
)
from .validation_block_yaml import ValidationBlockYaml
+ from .validation_block_yaml_data_schema import ValidationBlockYamlDataSchema
from .validation_error import ValidationError
from .validation_error_loc_item import ValidationErrorLocItem
from .wait_block import WaitBlock
@@ -1084,6 +1085,7 @@ _dynamic_imports: typing.Dict[str, str] = {
"ValidationBlockParametersItem_Output": ".validation_block_parameters_item",
"ValidationBlockParametersItem_Workflow": ".validation_block_parameters_item",
"ValidationBlockYaml": ".validation_block_yaml",
+ "ValidationBlockYamlDataSchema": ".validation_block_yaml_data_schema",
"ValidationError": ".validation_error",
"ValidationErrorLocItem": ".validation_error_loc_item",
"WaitBlock": ".wait_block",
@@ -1715,6 +1717,7 @@ __all__ = [
"ValidationBlockParametersItem_Output",
"ValidationBlockParametersItem_Workflow",
"ValidationBlockYaml",
+ "ValidationBlockYamlDataSchema",
"ValidationError",
"ValidationErrorLocItem",
"WaitBlock",
diff --git a/skyvern/client/types/for_loop_block_yaml_loop_blocks_item.py b/skyvern/client/types/for_loop_block_yaml_loop_blocks_item.py
index 7a87b74ec..17984a9de 100644
--- a/skyvern/client/types/for_loop_block_yaml_loop_blocks_item.py
+++ b/skyvern/client/types/for_loop_block_yaml_loop_blocks_item.py
@@ -17,6 +17,7 @@ from .for_loop_block_yaml_data_schema import ForLoopBlockYamlDataSchema
from .pdf_format import PdfFormat
from .run_engine import RunEngine
from .task_block_yaml_data_schema import TaskBlockYamlDataSchema
+from .validation_block_yaml_data_schema import ValidationBlockYamlDataSchema
class ForLoopBlockYamlLoopBlocksItem_Task(UniversalBaseModel):
@@ -245,11 +246,14 @@ class ForLoopBlockYamlLoopBlocksItem_Validation(UniversalBaseModel):
continue_on_failure: typing.Optional[bool] = None
model: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = None
next_loop_on_failure: typing.Optional[bool] = None
+ navigation_goal: typing.Optional[str] = None
complete_criterion: typing.Optional[str] = None
terminate_criterion: typing.Optional[str] = None
error_code_mapping: typing.Optional[typing.Dict[str, typing.Optional[str]]] = None
parameter_keys: typing.Optional[typing.List[str]] = None
disable_cache: typing.Optional[bool] = None
+ data_extraction_goal: typing.Optional[str] = None
+ data_schema: typing.Optional[ValidationBlockYamlDataSchema] = None
if IS_PYDANTIC_V2:
model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
diff --git a/skyvern/client/types/validation_block_yaml.py b/skyvern/client/types/validation_block_yaml.py
index 4bdd9434d..86ae09c1e 100644
--- a/skyvern/client/types/validation_block_yaml.py
+++ b/skyvern/client/types/validation_block_yaml.py
@@ -4,6 +4,7 @@ import typing
import pydantic
from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
+from .validation_block_yaml_data_schema import ValidationBlockYamlDataSchema
class ValidationBlockYaml(UniversalBaseModel):
@@ -20,11 +21,14 @@ class ValidationBlockYaml(UniversalBaseModel):
continue_on_failure: typing.Optional[bool] = None
model: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = None
next_loop_on_failure: typing.Optional[bool] = None
+ navigation_goal: typing.Optional[str] = None
complete_criterion: typing.Optional[str] = None
terminate_criterion: typing.Optional[str] = None
error_code_mapping: typing.Optional[typing.Dict[str, typing.Optional[str]]] = None
parameter_keys: typing.Optional[typing.List[str]] = None
disable_cache: typing.Optional[bool] = None
+ data_extraction_goal: typing.Optional[str] = None
+ data_schema: typing.Optional[ValidationBlockYamlDataSchema] = None
if IS_PYDANTIC_V2:
model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
diff --git a/skyvern/client/types/validation_block_yaml_data_schema.py b/skyvern/client/types/validation_block_yaml_data_schema.py
new file mode 100644
index 000000000..da5435480
--- /dev/null
+++ b/skyvern/client/types/validation_block_yaml_data_schema.py
@@ -0,0 +1,7 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import typing
+
+ValidationBlockYamlDataSchema = typing.Union[
+ typing.Dict[str, typing.Optional[typing.Any]], typing.List[typing.Optional[typing.Any]], str
+]
diff --git a/skyvern/client/types/while_loop_block_yaml_loop_blocks_item.py b/skyvern/client/types/while_loop_block_yaml_loop_blocks_item.py
index c604507f9..16d9cd25b 100644
--- a/skyvern/client/types/while_loop_block_yaml_loop_blocks_item.py
+++ b/skyvern/client/types/while_loop_block_yaml_loop_blocks_item.py
@@ -17,6 +17,7 @@ from .for_loop_block_yaml_data_schema import ForLoopBlockYamlDataSchema
from .pdf_format import PdfFormat
from .run_engine import RunEngine
from .task_block_yaml_data_schema import TaskBlockYamlDataSchema
+from .validation_block_yaml_data_schema import ValidationBlockYamlDataSchema
class WhileLoopBlockYamlLoopBlocksItem_Task(UniversalBaseModel):
@@ -245,11 +246,14 @@ class WhileLoopBlockYamlLoopBlocksItem_Validation(UniversalBaseModel):
continue_on_failure: typing.Optional[bool] = None
model: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = None
next_loop_on_failure: typing.Optional[bool] = None
+ navigation_goal: typing.Optional[str] = None
complete_criterion: typing.Optional[str] = None
terminate_criterion: typing.Optional[str] = None
error_code_mapping: typing.Optional[typing.Dict[str, typing.Optional[str]]] = None
parameter_keys: typing.Optional[typing.List[str]] = None
disable_cache: typing.Optional[bool] = None
+ data_extraction_goal: typing.Optional[str] = None
+ data_schema: typing.Optional[ValidationBlockYamlDataSchema] = None
if IS_PYDANTIC_V2:
model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
diff --git a/skyvern/client/types/workflow_definition_yaml_blocks_item.py b/skyvern/client/types/workflow_definition_yaml_blocks_item.py
index e43eee71b..a90521d64 100644
--- a/skyvern/client/types/workflow_definition_yaml_blocks_item.py
+++ b/skyvern/client/types/workflow_definition_yaml_blocks_item.py
@@ -17,6 +17,7 @@ from .for_loop_block_yaml_data_schema import ForLoopBlockYamlDataSchema
from .pdf_format import PdfFormat
from .run_engine import RunEngine
from .task_block_yaml_data_schema import TaskBlockYamlDataSchema
+from .validation_block_yaml_data_schema import ValidationBlockYamlDataSchema
class WorkflowDefinitionYamlBlocksItem_Action(UniversalBaseModel):
@@ -562,11 +563,14 @@ class WorkflowDefinitionYamlBlocksItem_Validation(UniversalBaseModel):
continue_on_failure: typing.Optional[bool] = None
model: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = None
next_loop_on_failure: typing.Optional[bool] = None
+ navigation_goal: typing.Optional[str] = None
complete_criterion: typing.Optional[str] = None
terminate_criterion: typing.Optional[str] = None
error_code_mapping: typing.Optional[typing.Dict[str, typing.Optional[str]]] = None
parameter_keys: typing.Optional[typing.List[str]] = None
disable_cache: typing.Optional[bool] = None
+ data_extraction_goal: typing.Optional[str] = None
+ data_schema: typing.Optional[ValidationBlockYamlDataSchema] = None
if IS_PYDANTIC_V2:
model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
diff --git a/skyvern/forge/sdk/workflow/workflow_definition_converter.py b/skyvern/forge/sdk/workflow/workflow_definition_converter.py
index aded77aba..081ec5b9c 100644
--- a/skyvern/forge/sdk/workflow/workflow_definition_converter.py
+++ b/skyvern/forge/sdk/workflow/workflow_definition_converter.py
@@ -580,13 +580,23 @@ def block_yaml_to_block(
f"Validation block '{block_yaml.label}' requires at least one of complete_criterion or terminate_criterion"
)
+ if block_yaml.data_extraction_goal and not block_yaml.navigation_goal:
+ raise InvalidWorkflowDefinition(
+ f"Validation block '{block_yaml.label}' has data_extraction_goal but no navigation_goal; "
+ "extraction requires both fields to be set"
+ )
+
return ValidationBlock(
**base_kwargs,
task_type=TaskType.validation,
parameters=validation_block_parameters,
+ navigation_goal=block_yaml.navigation_goal,
complete_criterion=block_yaml.complete_criterion,
terminate_criterion=block_yaml.terminate_criterion,
error_code_mapping=block_yaml.error_code_mapping,
+ data_extraction_goal=block_yaml.data_extraction_goal,
+ data_schema=block_yaml.data_schema,
+ disable_cache=block_yaml.disable_cache,
# Should only need one step for validation block, but we allow 2 in case the LLM has an unexpected failure and we need to retry.
max_steps_per_run=2,
)
diff --git a/skyvern/schemas/workflows.py b/skyvern/schemas/workflows.py
index 7f3e30324..690884a26 100644
--- a/skyvern/schemas/workflows.py
+++ b/skyvern/schemas/workflows.py
@@ -879,11 +879,14 @@ class PDFParserBlockYAML(BlockYAML):
class ValidationBlockYAML(BlockYAML):
block_type: Literal[BlockType.VALIDATION] = BlockType.VALIDATION # type: ignore
+ navigation_goal: str | None = None
complete_criterion: str | None = None
terminate_criterion: str | None = None
error_code_mapping: dict[str, str] | None = None
parameter_keys: list[str] | None = None
disable_cache: bool = False
+ data_extraction_goal: str | None = None
+ data_schema: dict[str, Any] | list | str | None = None
class ActionBlockYAML(BlockYAML):
diff --git a/tests/unit/test_validation_block_extraction.py b/tests/unit/test_validation_block_extraction.py
new file mode 100644
index 000000000..bdfac0685
--- /dev/null
+++ b/tests/unit/test_validation_block_extraction.py
@@ -0,0 +1,228 @@
+from datetime import datetime, timezone
+from typing import Any, TypeAlias, cast
+
+import pytest
+
+from skyvern.client.types.for_loop_block_yaml_loop_blocks_item import (
+ ForLoopBlockYamlLoopBlocksItem_Validation,
+)
+from skyvern.client.types.validation_block_yaml import ValidationBlockYaml
+from skyvern.client.types.while_loop_block_yaml_loop_blocks_item import (
+ WhileLoopBlockYamlLoopBlocksItem_Validation,
+)
+from skyvern.client.types.workflow_definition_yaml_blocks_item import (
+ WorkflowDefinitionYamlBlocksItem_Validation,
+)
+from skyvern.forge.agent import ForgeAgent
+from skyvern.forge.sdk.schemas.tasks import TaskType
+from skyvern.forge.sdk.workflow.exceptions import InvalidWorkflowDefinition
+from skyvern.forge.sdk.workflow.models.block import ValidationBlock
+from skyvern.forge.sdk.workflow.models.parameter import (
+ PARAMETER_TYPE,
+ OutputParameter,
+ ParameterType,
+)
+from skyvern.forge.sdk.workflow.workflow_definition_converter import block_yaml_to_block
+from skyvern.schemas.workflows import ValidationBlockYAML
+from skyvern.webeye.actions.actions import CompleteAction, TerminateAction
+from skyvern.webeye.actions.models import DetailedAgentStepOutput
+from skyvern.webeye.actions.responses import ActionFailure, ActionSuccess
+
+
+def _output_parameter(label: str) -> OutputParameter:
+ now = datetime.now(timezone.utc)
+ return OutputParameter(
+ parameter_type=ParameterType.OUTPUT,
+ key=f"{label}_output",
+ description="test output",
+ output_parameter_id=f"op_{label}",
+ workflow_id="w_test",
+ created_at=now,
+ modified_at=now,
+ )
+
+
+class TestValidationBlockExtractionFields:
+ def test_yaml_accepts_extraction_fields(self) -> None:
+ schema = {"type": "object", "properties": {"price": {"type": "number"}}}
+ block_yaml = ValidationBlockYAML(
+ label="validate_cart",
+ navigation_goal="validate the cart total",
+ complete_criterion="cart total matches expected total",
+ data_extraction_goal="extract the cart subtotal as price",
+ data_schema=schema,
+ )
+
+ assert block_yaml.navigation_goal == "validate the cart total"
+ assert block_yaml.data_extraction_goal == "extract the cart subtotal as price"
+ assert block_yaml.data_schema == schema
+
+ def test_yaml_defaults_extraction_fields_to_none(self) -> None:
+ block_yaml = ValidationBlockYAML(
+ label="validate_cart",
+ complete_criterion="cart total matches expected total",
+ )
+
+ assert block_yaml.navigation_goal is None
+ assert block_yaml.data_extraction_goal is None
+ assert block_yaml.data_schema is None
+
+ def test_converter_passes_extraction_fields_to_validation_block(self) -> None:
+ schema = {"type": "object", "properties": {"price": {"type": "number"}}}
+ block_yaml = ValidationBlockYAML(
+ label="validate_cart",
+ navigation_goal="validate the cart total",
+ terminate_criterion="cart is empty",
+ data_extraction_goal="extract the cart subtotal as price",
+ data_schema=schema,
+ )
+ parameters: dict[str, PARAMETER_TYPE] = {
+ "validate_cart_output": cast(PARAMETER_TYPE, _output_parameter("validate_cart")),
+ }
+
+ block = block_yaml_to_block(block_yaml, parameters)
+
+ assert isinstance(block, ValidationBlock)
+ assert block.navigation_goal == "validate the cart total"
+ assert block.data_extraction_goal == "extract the cart subtotal as price"
+ assert block.data_schema == schema
+ assert block.terminate_criterion == "cart is empty"
+ assert block.disable_cache is False
+
+ def test_converter_threads_disable_cache_to_validation_block(self) -> None:
+ block_yaml = ValidationBlockYAML(
+ label="validate_cart",
+ complete_criterion="cart total matches expected total",
+ disable_cache=True,
+ )
+ parameters: dict[str, PARAMETER_TYPE] = {
+ "validate_cart_output": cast(PARAMETER_TYPE, _output_parameter("validate_cart")),
+ }
+
+ block = block_yaml_to_block(block_yaml, parameters)
+
+ assert isinstance(block, ValidationBlock)
+ assert block.disable_cache is True
+
+ def test_converter_rejects_extraction_goal_without_navigation_goal(self) -> None:
+ block_yaml = ValidationBlockYAML(
+ label="validate_cart",
+ complete_criterion="cart total matches expected total",
+ data_extraction_goal="extract the cart subtotal as price",
+ )
+ parameters: dict[str, PARAMETER_TYPE] = {
+ "validate_cart_output": cast(PARAMETER_TYPE, _output_parameter("validate_cart")),
+ }
+
+ with pytest.raises(InvalidWorkflowDefinition, match="navigation_goal"):
+ block_yaml_to_block(block_yaml, parameters)
+
+ def test_yaml_roundtrip_preserves_extraction_fields(self) -> None:
+ schema = {"type": "object", "properties": {"price": {"type": "number"}}}
+ original = ValidationBlockYAML(
+ label="validate_cart",
+ navigation_goal="validate the cart total",
+ complete_criterion="cart total matches expected total",
+ terminate_criterion="cart is empty",
+ data_extraction_goal="extract the cart subtotal as price",
+ data_schema=schema,
+ error_code_mapping={"NO_CART": "cart is missing"},
+ parameter_keys=["expected_total"],
+ disable_cache=True,
+ )
+
+ dumped = original.model_dump()
+ restored = ValidationBlockYAML.model_validate(dumped)
+
+ assert restored.navigation_goal == original.navigation_goal
+ assert restored.data_extraction_goal == original.data_extraction_goal
+ assert restored.data_schema == original.data_schema
+ assert restored.complete_criterion == original.complete_criterion
+ assert restored.terminate_criterion == original.terminate_criterion
+ assert restored.error_code_mapping == original.error_code_mapping
+ assert restored.parameter_keys == original.parameter_keys
+ assert restored.disable_cache == original.disable_cache
+
+
+FernValidationYamlModel: TypeAlias = (
+ type[ValidationBlockYaml]
+ | type[WorkflowDefinitionYamlBlocksItem_Validation]
+ | type[ForLoopBlockYamlLoopBlocksItem_Validation]
+ | type[WhileLoopBlockYamlLoopBlocksItem_Validation]
+)
+
+
+class TestValidationBlockFernSdkFields:
+ def test_fern_validation_yaml_models_accept_runtime_extraction_fields(self) -> None:
+ schema: dict[str, Any] = {
+ "type": "object",
+ "properties": {"price": {"type": "number"}},
+ }
+ model_classes: tuple[FernValidationYamlModel, ...] = (
+ ValidationBlockYaml,
+ WorkflowDefinitionYamlBlocksItem_Validation,
+ ForLoopBlockYamlLoopBlocksItem_Validation,
+ WhileLoopBlockYamlLoopBlocksItem_Validation,
+ )
+
+ for model_class in model_classes:
+ block_yaml = model_class(
+ label="validate_cart",
+ navigation_goal="validate the cart total",
+ complete_criterion="cart total matches expected total",
+ data_extraction_goal="extract the cart subtotal as price",
+ data_schema=schema,
+ )
+
+ assert block_yaml.navigation_goal == "validate the cart total"
+ assert block_yaml.data_extraction_goal == "extract the cart subtotal as price"
+ assert block_yaml.data_schema == schema
+
+
+class TestValidationBlockRuntimeExtractionGate:
+ def test_converter_output_satisfies_runtime_extraction_gate(self) -> None:
+ schema = {"type": "object", "properties": {"price": {"type": "number"}}}
+ block_yaml = ValidationBlockYAML(
+ label="validate_cart",
+ navigation_goal="validate the cart total",
+ complete_criterion="cart total matches expected total",
+ data_extraction_goal="extract the cart subtotal as price",
+ data_schema=schema,
+ )
+ parameters: dict[str, PARAMETER_TYPE] = {
+ "validate_cart_output": cast(PARAMETER_TYPE, _output_parameter("validate_cart")),
+ }
+
+ block = block_yaml_to_block(block_yaml, parameters)
+
+ assert isinstance(block, ValidationBlock)
+ assert block.task_type == TaskType.validation
+ assert block.navigation_goal is not None
+ assert block.data_extraction_goal is not None
+ assert block.data_schema is not None
+
+ def test_step_has_completed_goal_true_on_successful_complete_action(self) -> None:
+ complete_action = CompleteAction(reasoning="criterion met", verified=True)
+ detailed_output = DetailedAgentStepOutput(
+ scraped_page=None,
+ extract_action_prompt=None,
+ llm_response=None,
+ actions=[complete_action],
+ action_results=[ActionSuccess()],
+ actions_and_results=[(complete_action, [ActionSuccess()])],
+ )
+
+ assert ForgeAgent.step_has_completed_goal(detailed_output) is True
+
+ def test_step_has_completed_goal_false_on_terminate_action(self) -> None:
+ terminate_action = TerminateAction(reasoning="cannot continue")
+ detailed_output = DetailedAgentStepOutput(
+ scraped_page=None,
+ extract_action_prompt=None,
+ llm_response=None,
+ actions=[terminate_action],
+ action_results=[ActionFailure(exception=Exception("terminated"))],
+ actions_and_results=[(terminate_action, [ActionFailure(exception=Exception("terminated"))])],
+ )
+
+ assert ForgeAgent.step_has_completed_goal(detailed_output) is False