Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(workflow): expose data_extraction_goal and data_schema on ValidationBlock backend (#6066)
Aaron Perez6ccca74c3ea7
diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.jsonindex 8b6847625..22e2ada20 100644--- a/docs/api-reference/openapi.json+++ b/docs/api-reference/openapi.json@@ -12059,6 +12059,11 @@"title": "Next Loop On Failure","default": false},+ "navigation_goal": {+ "type": "string",+ "nullable": true,+ "title": "Navigation Goal"+ },"complete_criterion": {"type": "string","nullable": true,@@ -12089,6 +12094,28 @@"type": "boolean","title": "Disable Cache","default": false+ },+ "data_extraction_goal": {+ "type": "string",+ "nullable": true,+ "title": "Data Extraction Goal"+ },+ "data_schema": {+ "anyOf": [+ {+ "additionalProperties": true,+ "type": "object"+ },+ {+ "items": {},+ "type": "array"+ },+ {+ "type": "string"+ }+ ],+ "title": "Data Schema",+ "nullable": true}},"type": "object",diff --git a/fern/openapi/skyvern_openapi.json b/fern/openapi/skyvern_openapi.jsonindex 7e4193fed..8fe316071 100644--- a/fern/openapi/skyvern_openapi.json+++ b/fern/openapi/skyvern_openapi.json@@ -19164,6 +19164,17 @@"title": "Next Loop On Failure","default": false},+ "navigation_goal": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Navigation Goal"+ },"complete_criterion": {"anyOf": [{@@ -19218,6 +19229,36 @@"type": "boolean","title": "Disable Cache","default": false+ },+ "data_extraction_goal": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Data Extraction Goal"+ },+ "data_schema": {+ "anyOf": [+ {+ "additionalProperties": true,+ "type": "object"+ },+ {+ "items": {},+ "type": "array"+ },+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Data Schema"}},"type": "object",diff --git a/skyvern/client/types/__init__.py b/skyvern/client/types/__init__.pyindex 38799a4df..172687f95 100644--- a/skyvern/client/types/__init__.py+++ b/skyvern/client/types/__init__.py@@ -459,6 +459,7 @@ if typing.TYPE_CHECKING:ValidationBlockParametersItem_Workflow,)from .validation_block_yaml import ValidationBlockYaml+ from .validation_block_yaml_data_schema import ValidationBlockYamlDataSchemafrom .validation_error import ValidationErrorfrom .validation_error_loc_item import ValidationErrorLocItemfrom .wait_block import WaitBlock@@ -1084,6 +1085,7 @@ _dynamic_imports: typing.Dict[str, str] = {"ValidationBlockParametersItem_Output": ".validation_block_parameters_item","ValidationBlockParametersItem_Workflow": ".validation_block_parameters_item","ValidationBlockYaml": ".validation_block_yaml",+ "ValidationBlockYamlDataSchema": ".validation_block_yaml_data_schema","ValidationError": ".validation_error","ValidationErrorLocItem": ".validation_error_loc_item","WaitBlock": ".wait_block",@@ -1715,6 +1717,7 @@ __all__ = ["ValidationBlockParametersItem_Output","ValidationBlockParametersItem_Workflow","ValidationBlockYaml",+ "ValidationBlockYamlDataSchema","ValidationError","ValidationErrorLocItem","WaitBlock",diff --git a/skyvern/client/types/for_loop_block_yaml_loop_blocks_item.py b/skyvern/client/types/for_loop_block_yaml_loop_blocks_item.pyindex 7a87b74ec..17984a9de 100644--- a/skyvern/client/types/for_loop_block_yaml_loop_blocks_item.py+++ b/skyvern/client/types/for_loop_block_yaml_loop_blocks_item.py@@ -17,6 +17,7 @@ from .for_loop_block_yaml_data_schema import ForLoopBlockYamlDataSchemafrom .pdf_format import PdfFormatfrom .run_engine import RunEnginefrom .task_block_yaml_data_schema import TaskBlockYamlDataSchema+from .validation_block_yaml_data_schema import ValidationBlockYamlDataSchemaclass ForLoopBlockYamlLoopBlocksItem_Task(UniversalBaseModel):@@ -245,11 +246,14 @@ class ForLoopBlockYamlLoopBlocksItem_Validation(UniversalBaseModel):continue_on_failure: typing.Optional[bool] = Nonemodel: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = Nonenext_loop_on_failure: typing.Optional[bool] = None+ navigation_goal: typing.Optional[str] = Nonecomplete_criterion: typing.Optional[str] = Noneterminate_criterion: typing.Optional[str] = Noneerror_code_mapping: typing.Optional[typing.Dict[str, typing.Optional[str]]] = Noneparameter_keys: typing.Optional[typing.List[str]] = Nonedisable_cache: typing.Optional[bool] = None+ data_extraction_goal: typing.Optional[str] = None+ data_schema: typing.Optional[ValidationBlockYamlDataSchema] = Noneif IS_PYDANTIC_V2:model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2diff --git a/skyvern/client/types/validation_block_yaml.py b/skyvern/client/types/validation_block_yaml.pyindex 4bdd9434d..86ae09c1e 100644--- a/skyvern/client/types/validation_block_yaml.py+++ b/skyvern/client/types/validation_block_yaml.py@@ -4,6 +4,7 @@ import typingimport pydanticfrom ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel+from .validation_block_yaml_data_schema import ValidationBlockYamlDataSchemaclass ValidationBlockYaml(UniversalBaseModel):@@ -20,11 +21,14 @@ class ValidationBlockYaml(UniversalBaseModel):continue_on_failure: typing.Optional[bool] = Nonemodel: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = Nonenext_loop_on_failure: typing.Optional[bool] = None+ navigation_goal: typing.Optional[str] = Nonecomplete_criterion: typing.Optional[str] = Noneterminate_criterion: typing.Optional[str] = Noneerror_code_mapping: typing.Optional[typing.Dict[str, typing.Optional[str]]] = Noneparameter_keys: typing.Optional[typing.List[str]] = Nonedisable_cache: typing.Optional[bool] = None+ data_extraction_goal: typing.Optional[str] = None+ data_schema: typing.Optional[ValidationBlockYamlDataSchema] = Noneif IS_PYDANTIC_V2:model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2diff --git a/skyvern/client/types/validation_block_yaml_data_schema.py b/skyvern/client/types/validation_block_yaml_data_schema.pynew file mode 100644index 000000000..da5435480--- /dev/null+++ b/skyvern/client/types/validation_block_yaml_data_schema.py@@ -0,0 +1,7 @@+# This file was auto-generated by Fern from our API Definition.++import typing++ValidationBlockYamlDataSchema = typing.Union[+ typing.Dict[str, typing.Optional[typing.Any]], typing.List[typing.Optional[typing.Any]], str+]diff --git a/skyvern/client/types/while_loop_block_yaml_loop_blocks_item.py b/skyvern/client/types/while_loop_block_yaml_loop_blocks_item.pyindex c604507f9..16d9cd25b 100644--- a/skyvern/client/types/while_loop_block_yaml_loop_blocks_item.py+++ b/skyvern/client/types/while_loop_block_yaml_loop_blocks_item.py@@ -17,6 +17,7 @@ from .for_loop_block_yaml_data_schema import ForLoopBlockYamlDataSchemafrom .pdf_format import PdfFormatfrom .run_engine import RunEnginefrom .task_block_yaml_data_schema import TaskBlockYamlDataSchema+from .validation_block_yaml_data_schema import ValidationBlockYamlDataSchemaclass WhileLoopBlockYamlLoopBlocksItem_Task(UniversalBaseModel):@@ -245,11 +246,14 @@ class WhileLoopBlockYamlLoopBlocksItem_Validation(UniversalBaseModel):continue_on_failure: typing.Optional[bool] = Nonemodel: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = Nonenext_loop_on_failure: typing.Optional[bool] = None+ navigation_goal: typing.Optional[str] = Nonecomplete_criterion: typing.Optional[str] = Noneterminate_criterion: typing.Optional[str] = Noneerror_code_mapping: typing.Optional[typing.Dict[str, typing.Optional[str]]] = Noneparameter_keys: typing.Optional[typing.List[str]] = Nonedisable_cache: typing.Optional[bool] = None+ data_extraction_goal: typing.Optional[str] = None+ data_schema: typing.Optional[ValidationBlockYamlDataSchema] = Noneif IS_PYDANTIC_V2:model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2diff --git a/skyvern/client/types/workflow_definition_yaml_blocks_item.py b/skyvern/client/types/workflow_definition_yaml_blocks_item.pyindex e43eee71b..a90521d64 100644--- a/skyvern/client/types/workflow_definition_yaml_blocks_item.py+++ b/skyvern/client/types/workflow_definition_yaml_blocks_item.py@@ -17,6 +17,7 @@ from .for_loop_block_yaml_data_schema import ForLoopBlockYamlDataSchemafrom .pdf_format import PdfFormatfrom .run_engine import RunEnginefrom .task_block_yaml_data_schema import TaskBlockYamlDataSchema+from .validation_block_yaml_data_schema import ValidationBlockYamlDataSchemaclass WorkflowDefinitionYamlBlocksItem_Action(UniversalBaseModel):@@ -562,11 +563,14 @@ class WorkflowDefinitionYamlBlocksItem_Validation(UniversalBaseModel):continue_on_failure: typing.Optional[bool] = Nonemodel: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = Nonenext_loop_on_failure: typing.Optional[bool] = None+ navigation_goal: typing.Optional[str] = Nonecomplete_criterion: typing.Optional[str] = Noneterminate_criterion: typing.Optional[str] = Noneerror_code_mapping: typing.Optional[typing.Dict[str, typing.Optional[str]]] = Noneparameter_keys: typing.Optional[typing.List[str]] = Nonedisable_cache: typing.Optional[bool] = None+ data_extraction_goal: typing.Optional[str] = None+ data_schema: typing.Optional[ValidationBlockYamlDataSchema] = Noneif IS_PYDANTIC_V2:model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2diff --git a/skyvern/forge/sdk/workflow/workflow_definition_converter.py b/skyvern/forge/sdk/workflow/workflow_definition_converter.pyindex aded77aba..081ec5b9c 100644--- a/skyvern/forge/sdk/workflow/workflow_definition_converter.py+++ b/skyvern/forge/sdk/workflow/workflow_definition_converter.py@@ -580,13 +580,23 @@ def block_yaml_to_block(f"Validation block '{block_yaml.label}' requires at least one of complete_criterion or terminate_criterion")+ if block_yaml.data_extraction_goal and not block_yaml.navigation_goal:+ raise InvalidWorkflowDefinition(+ f"Validation block '{block_yaml.label}' has data_extraction_goal but no navigation_goal; "+ "extraction requires both fields to be set"+ )+return ValidationBlock(**base_kwargs,task_type=TaskType.validation,parameters=validation_block_parameters,+ navigation_goal=block_yaml.navigation_goal,complete_criterion=block_yaml.complete_criterion,terminate_criterion=block_yaml.terminate_criterion,error_code_mapping=block_yaml.error_code_mapping,+ data_extraction_goal=block_yaml.data_extraction_goal,+ data_schema=block_yaml.data_schema,+ disable_cache=block_yaml.disable_cache,# Should only need one step for validation block, but we allow 2 in case the LLM has an unexpected failure and we need to retry.max_steps_per_run=2,)diff --git a/skyvern/schemas/workflows.py b/skyvern/schemas/workflows.pyindex 7f3e30324..690884a26 100644--- a/skyvern/schemas/workflows.py+++ b/skyvern/schemas/workflows.py@@ -879,11 +879,14 @@ class PDFParserBlockYAML(BlockYAML):class ValidationBlockYAML(BlockYAML):block_type: Literal[BlockType.VALIDATION] = BlockType.VALIDATION # type: ignore+ navigation_goal: str | None = Nonecomplete_criterion: str | None = Noneterminate_criterion: str | None = Noneerror_code_mapping: dict[str, str] | None = Noneparameter_keys: list[str] | None = Nonedisable_cache: bool = False+ data_extraction_goal: str | None = None+ data_schema: dict[str, Any] | list | str | None = Noneclass ActionBlockYAML(BlockYAML):diff --git a/tests/unit/test_validation_block_extraction.py b/tests/unit/test_validation_block_extraction.pynew file mode 100644index 000000000..bdfac0685--- /dev/null+++ b/tests/unit/test_validation_block_extraction.py@@ -0,0 +1,228 @@+from datetime import datetime, timezone+from typing import Any, TypeAlias, cast++import pytest++from skyvern.client.types.for_loop_block_yaml_loop_blocks_item import (+ ForLoopBlockYamlLoopBlocksItem_Validation,+)+from skyvern.client.types.validation_block_yaml import ValidationBlockYaml+from skyvern.client.types.while_loop_block_yaml_loop_blocks_item import (+ WhileLoopBlockYamlLoopBlocksItem_Validation,+)+from skyvern.client.types.workflow_definition_yaml_blocks_item import (+ WorkflowDefinitionYamlBlocksItem_Validation,+)+from skyvern.forge.agent import ForgeAgent+from skyvern.forge.sdk.schemas.tasks import TaskType+from skyvern.forge.sdk.workflow.exceptions import InvalidWorkflowDefinition+from skyvern.forge.sdk.workflow.models.block import ValidationBlock+from skyvern.forge.sdk.workflow.models.parameter import (+ PARAMETER_TYPE,+ OutputParameter,+ ParameterType,+)+from skyvern.forge.sdk.workflow.workflow_definition_converter import block_yaml_to_block+from skyvern.schemas.workflows import ValidationBlockYAML+from skyvern.webeye.actions.actions import CompleteAction, TerminateAction+from skyvern.webeye.actions.models import DetailedAgentStepOutput+from skyvern.webeye.actions.responses import ActionFailure, ActionSuccess+++def _output_parameter(label: str) -> OutputParameter:+ now = datetime.now(timezone.utc)+ return OutputParameter(+ parameter_type=ParameterType.OUTPUT,+ key=f"{label}_output",+ description="test output",+ output_parameter_id=f"op_{label}",+ workflow_id="w_test",+ created_at=now,+ modified_at=now,+ )+++class TestValidationBlockExtractionFields:+ def test_yaml_accepts_extraction_fields(self) -> None:+ schema = {"type": "object", "properties": {"price": {"type": "number"}}}+ block_yaml = ValidationBlockYAML(+ label="validate_cart",+ navigation_goal="validate the cart total",+ complete_criterion="cart total matches expected total",+ data_extraction_goal="extract the cart subtotal as price",+ data_schema=schema,+ )++ assert block_yaml.navigation_goal == "validate the cart total"+ assert block_yaml.data_extraction_goal == "extract the cart subtotal as price"+ assert block_yaml.data_schema == schema++ def test_yaml_defaults_extraction_fields_to_none(self) -> None:+ block_yaml = ValidationBlockYAML(+ label="validate_cart",+ complete_criterion="cart total matches expected total",+ )++ assert block_yaml.navigation_goal is None+ assert block_yaml.data_extraction_goal is None+ assert block_yaml.data_schema is None++ def test_converter_passes_extraction_fields_to_validation_block(self) -> None:+ schema = {"type": "object", "properties": {"price": {"type": "number"}}}+ block_yaml = ValidationBlockYAML(+ label="validate_cart",+ navigation_goal="validate the cart total",+ terminate_criterion="cart is empty",+ data_extraction_goal="extract the cart subtotal as price",+ data_schema=schema,+ )+ parameters: dict[str, PARAMETER_TYPE] = {+ "validate_cart_output": cast(PARAMETER_TYPE, _output_parameter("validate_cart")),+ }++ block = block_yaml_to_block(block_yaml, parameters)++ assert isinstance(block, ValidationBlock)+ assert block.navigation_goal == "validate the cart total"+ assert block.data_extraction_goal == "extract the cart subtotal as price"+ assert block.data_schema == schema+ assert block.terminate_criterion == "cart is empty"+ assert block.disable_cache is False++ def test_converter_threads_disable_cache_to_validation_block(self) -> None:+ block_yaml = ValidationBlockYAML(+ label="validate_cart",+ complete_criterion="cart total matches expected total",+ disable_cache=True,+ )+ parameters: dict[str, PARAMETER_TYPE] = {+ "validate_cart_output": cast(PARAMETER_TYPE, _output_parameter("validate_cart")),+ }++ block = block_yaml_to_block(block_yaml, parameters)++ assert isinstance(block, ValidationBlock)+ assert block.disable_cache is True++ def test_converter_rejects_extraction_goal_without_navigation_goal(self) -> None:+ block_yaml = ValidationBlockYAML(+ label="validate_cart",+ complete_criterion="cart total matches expected total",+ data_extraction_goal="extract the cart subtotal as price",+ )+ parameters: dict[str, PARAMETER_TYPE] = {+ "validate_cart_output": cast(PARAMETER_TYPE, _output_parameter("validate_cart")),+ }++ with pytest.raises(InvalidWorkflowDefinition, match="navigation_goal"):+ block_yaml_to_block(block_yaml, parameters)++ def test_yaml_roundtrip_preserves_extraction_fields(self) -> None:+ schema = {"type": "object", "properties": {"price": {"type": "number"}}}+ original = ValidationBlockYAML(+ label="validate_cart",+ navigation_goal="validate the cart total",+ complete_criterion="cart total matches expected total",+ terminate_criterion="cart is empty",+ data_extraction_goal="extract the cart subtotal as price",+ data_schema=schema,+ error_code_mapping={"NO_CART": "cart is missing"},+ parameter_keys=["expected_total"],+ disable_cache=True,+ )++ dumped = original.model_dump()+ restored = ValidationBlockYAML.model_validate(dumped)++ assert restored.navigation_goal == original.navigation_goal+ assert restored.data_extraction_goal == original.data_extraction_goal+ assert restored.data_schema == original.data_schema+ assert restored.complete_criterion == original.complete_criterion+ assert restored.terminate_criterion == original.terminate_criterion+ assert restored.error_code_mapping == original.error_code_mapping+ assert restored.parameter_keys == original.parameter_keys+ assert restored.disable_cache == original.disable_cache+++FernValidationYamlModel: TypeAlias = (+ type[ValidationBlockYaml]+ | type[WorkflowDefinitionYamlBlocksItem_Validation]+ | type[ForLoopBlockYamlLoopBlocksItem_Validation]+ | type[WhileLoopBlockYamlLoopBlocksItem_Validation]+)+++class TestValidationBlockFernSdkFields:+ def test_fern_validation_yaml_models_accept_runtime_extraction_fields(self) -> None:+ schema: dict[str, Any] = {+ "type": "object",+ "properties": {"price": {"type": "number"}},+ }+ model_classes: tuple[FernValidationYamlModel, ...] = (+ ValidationBlockYaml,+ WorkflowDefinitionYamlBlocksItem_Validation,+ ForLoopBlockYamlLoopBlocksItem_Validation,+ WhileLoopBlockYamlLoopBlocksItem_Validation,+ )++ for model_class in model_classes:+ block_yaml = model_class(+ label="validate_cart",+ navigation_goal="validate the cart total",+ complete_criterion="cart total matches expected total",+ data_extraction_goal="extract the cart subtotal as price",+ data_schema=schema,+ )++ assert block_yaml.navigation_goal == "validate the cart total"+ assert block_yaml.data_extraction_goal == "extract the cart subtotal as price"+ assert block_yaml.data_schema == schema+++class TestValidationBlockRuntimeExtractionGate:+ def test_converter_output_satisfies_runtime_extraction_gate(self) -> None:+ schema = {"type": "object", "properties": {"price": {"type": "number"}}}+ block_yaml = ValidationBlockYAML(+ label="validate_cart",+ navigation_goal="validate the cart total",+ complete_criterion="cart total matches expected total",+ data_extraction_goal="extract the cart subtotal as price",+ data_schema=schema,+ )+ parameters: dict[str, PARAMETER_TYPE] = {+ "validate_cart_output": cast(PARAMETER_TYPE, _output_parameter("validate_cart")),+ }++ block = block_yaml_to_block(block_yaml, parameters)++ assert isinstance(block, ValidationBlock)+ assert block.task_type == TaskType.validation+ assert block.navigation_goal is not None+ assert block.data_extraction_goal is not None+ assert block.data_schema is not None++ def test_step_has_completed_goal_true_on_successful_complete_action(self) -> None:+ complete_action = CompleteAction(reasoning="criterion met", verified=True)+ detailed_output = DetailedAgentStepOutput(+ scraped_page=None,+ extract_action_prompt=None,+ llm_response=None,+ actions=[complete_action],+ action_results=[ActionSuccess()],+ actions_and_results=[(complete_action, [ActionSuccess()])],+ )++ assert ForgeAgent.step_has_completed_goal(detailed_output) is True++ def test_step_has_completed_goal_false_on_terminate_action(self) -> None:+ terminate_action = TerminateAction(reasoning="cannot continue")+ detailed_output = DetailedAgentStepOutput(+ scraped_page=None,+ extract_action_prompt=None,+ llm_response=None,+ actions=[terminate_action],+ action_results=[ActionFailure(exception=Exception("terminated"))],+ actions_and_results=[(terminate_action, [ActionFailure(exception=Exception("terminated"))])],+ )++ assert ForgeAgent.step_has_completed_goal(detailed_output) is False