Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(SKY-9506): add selector + ai_fallback fields to Action Block schema (#6212)
Trevor Cheung80b6f67edec4
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex 21196fd9e..93b109ef5 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -124,6 +124,7 @@ from skyvern.forge.sdk.workflow.models.parameter import ()from skyvern.schemas.runs import RunEnginefrom skyvern.schemas.workflows import (+ AIFallbackMode,BlockResult,BlockStatus,BlockType,@@ -5786,6 +5787,9 @@ class ActionBlock(BaseTaskBlock):# Parameter 1 of Literal[...] cannot be of type "Any"block_type: Literal[BlockType.ACTION] = BlockType.ACTION # type: ignore+ selector: str | None = None+ ai_fallback: AIFallbackMode = AIFallbackMode.FALLBACK+class NavigationBlock(BaseTaskBlock):# There is a mypy bug with Literal. Without the type: ignore, mypy will raise an error:diff --git a/skyvern/forge/sdk/workflow/workflow_definition_converter.py b/skyvern/forge/sdk/workflow/workflow_definition_converter.pyindex aded77aba..4e5c4af26 100644--- a/skyvern/forge/sdk/workflow/workflow_definition_converter.py+++ b/skyvern/forge/sdk/workflow/workflow_definition_converter.py@@ -605,6 +605,8 @@ def block_yaml_to_block(task_type=TaskType.action,parameters=action_block_parameters,navigation_goal=block_yaml.navigation_goal,+ selector=block_yaml.selector,+ ai_fallback=block_yaml.ai_fallback,error_code_mapping=block_yaml.error_code_mapping,max_retries=block_yaml.max_retries,complete_on_download=block_yaml.complete_on_download,diff --git a/skyvern/schemas/workflows.py b/skyvern/schemas/workflows.pyindex f11b4b469..04ae3d1b5 100644--- a/skyvern/schemas/workflows.py+++ b/skyvern/schemas/workflows.py@@ -443,6 +443,20 @@ class BlockType(StrEnum):GOOGLE_SHEETS_WRITE = "google_sheets_write"+class AIFallbackMode(StrEnum):+ """Controls how an action block uses a CSS/XPath selector relative to AI.++ - `fallback`: attempt the selector first; fall back to AI on failure.+ - `proactive`: always use AI; the selector (if any) is a hint only.++ When `selector` is None, both modes degrade to AI-only — the difference+ is purely semantic (whether the author expected to have a selector here).+ """++ FALLBACK = "fallback"+ PROACTIVE = "proactive"++class BlockStatus(StrEnum):running = "running"completed = "completed"@@ -922,6 +936,8 @@ class ActionBlockYAML(BlockYAML):title: str = ""engine: RunEngine = RunEngine.skyvern_v1navigation_goal: str | None = None+ selector: str | None = None+ ai_fallback: AIFallbackMode = AIFallbackMode.FALLBACKerror_code_mapping: dict[str, str] | None = Nonemax_retries: int = 0parameter_keys: list[str] | None = None