Skip to content

Automate browser based workflows with AI

feat(SKY-9506): add selector + ai_fallback fields to Action Block schema (#6212)

Trevor Cheung80b6f67edec4
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index 21196fd9e..93b109ef5 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -124,6 +124,7 @@ from skyvern.forge.sdk.workflow.models.parameter import (
)
from skyvern.schemas.runs import RunEngine
from skyvern.schemas.workflows import (
+ AIFallbackMode,
BlockResult,
BlockStatus,
BlockType,
@@ -5786,6 +5787,9 @@ class ActionBlock(BaseTaskBlock):
# Parameter 1 of Literal[...] cannot be of type "Any"
block_type: Literal[BlockType.ACTION] = BlockType.ACTION # type: ignore
+ selector: str | None = None
+ ai_fallback: AIFallbackMode = AIFallbackMode.FALLBACK
+
class NavigationBlock(BaseTaskBlock):
# There is a mypy bug with Literal. Without the type: ignore, mypy will raise an error:
diff --git a/skyvern/forge/sdk/workflow/workflow_definition_converter.py b/skyvern/forge/sdk/workflow/workflow_definition_converter.py
index aded77aba..4e5c4af26 100644
--- a/skyvern/forge/sdk/workflow/workflow_definition_converter.py
+++ b/skyvern/forge/sdk/workflow/workflow_definition_converter.py
@@ -605,6 +605,8 @@ def block_yaml_to_block(
task_type=TaskType.action,
parameters=action_block_parameters,
navigation_goal=block_yaml.navigation_goal,
+ selector=block_yaml.selector,
+ ai_fallback=block_yaml.ai_fallback,
error_code_mapping=block_yaml.error_code_mapping,
max_retries=block_yaml.max_retries,
complete_on_download=block_yaml.complete_on_download,
diff --git a/skyvern/schemas/workflows.py b/skyvern/schemas/workflows.py
index f11b4b469..04ae3d1b5 100644
--- a/skyvern/schemas/workflows.py
+++ b/skyvern/schemas/workflows.py
@@ -443,6 +443,20 @@ class BlockType(StrEnum):
GOOGLE_SHEETS_WRITE = "google_sheets_write"
+class AIFallbackMode(StrEnum):
+ """Controls how an action block uses a CSS/XPath selector relative to AI.
+
+ - `fallback`: attempt the selector first; fall back to AI on failure.
+ - `proactive`: always use AI; the selector (if any) is a hint only.
+
+ When `selector` is None, both modes degrade to AI-only — the difference
+ is purely semantic (whether the author expected to have a selector here).
+ """
+
+ FALLBACK = "fallback"
+ PROACTIVE = "proactive"
+
+
class BlockStatus(StrEnum):
running = "running"
completed = "completed"
@@ -922,6 +936,8 @@ class ActionBlockYAML(BlockYAML):
title: str = ""
engine: RunEngine = RunEngine.skyvern_v1
navigation_goal: str | None = None
+ selector: str | None = None
+ ai_fallback: AIFallbackMode = AIFallbackMode.FALLBACK
error_code_mapping: dict[str, str] | None = None
max_retries: int = 0
parameter_keys: list[str] | None = None