Skip to content

Automate browser based workflows with AI

fix(SKY-8517): rich action data for script reviewer from AI fallback episodes (#5218)

pedrohsdb82628cdd2f66
diff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.py
index 79d6d753d..55c3783eb 100644
--- a/skyvern/forge/sdk/routes/agent_protocol.py
+++ b/skyvern/forge/sdk/routes/agent_protocol.py
@@ -1082,6 +1082,7 @@ async def delete_workflow(
"/folders",
response_model=Folder,
tags=["Workflow Folders"],
+ include_in_schema=False,
openapi_extra={
"x-fern-sdk-method-name": "create_folder",
},
@@ -1124,6 +1125,7 @@ async def create_folder(
"/folders/{folder_id}",
response_model=Folder,
tags=["Workflow Folders"],
+ include_in_schema=False,
openapi_extra={
"x-fern-sdk-method-name": "get_folder",
},
@@ -1168,6 +1170,7 @@ async def get_folder(
"/folders",
response_model=list[Folder],
tags=["Workflow Folders"],
+ include_in_schema=False,
openapi_extra={
"x-fern-sdk-method-name": "get_folders",
},
@@ -1225,6 +1228,7 @@ async def get_folders(
"/folders/{folder_id}",
response_model=Folder,
tags=["Workflow Folders"],
+ include_in_schema=False,
openapi_extra={
"x-fern-sdk-method-name": "update_folder",
},
@@ -1271,6 +1275,7 @@ async def update_folder(
@base_router.delete(
"/folders/{folder_id}",
tags=["Workflow Folders"],
+ include_in_schema=False,
openapi_extra={
"x-fern-sdk-method-name": "delete_folder",
},
@@ -1307,6 +1312,7 @@ async def delete_folder(
"/workflows/{workflow_permanent_id}/folder",
response_model=Workflow,
tags=["Workflow Folders"],
+ include_in_schema=False,
openapi_extra={
"x-fern-sdk-method-name": "update_workflow_folder",
},
@@ -1397,6 +1403,7 @@ async def convert_curl_to_http(
"/artifacts/{artifact_id}",
tags=["Artifacts"],
response_model=Artifact,
+ include_in_schema=False,
openapi_extra={
"x-fern-sdk-method-name": "get_artifact",
},
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 4ef40fc73..1c2f13434 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -121,8 +121,7 @@ from skyvern.schemas.workflows import (
WorkflowStatus,
)
from skyvern.services import script_service, workflow_script_service
-from skyvern.utils.css_selector import compute_stable_selector
-from skyvern.webeye.actions.actions import Action
+from skyvern.utils.css_selector import build_action_summary # shared with script_service
from skyvern.webeye.browser_state import BrowserState
LOG = structlog.get_logger()
@@ -290,69 +289,6 @@ def _get_workflow_definition_core_data(workflow_definition: WorkflowDefinition)
return workflow_dict
-# Attributes safe to pass to the script reviewer (excludes noisy/dynamic attrs)
-_REVIEWER_SAFE_ATTRS = frozenset(
- {
- "name",
- "id",
- "placeholder",
- "aria-label",
- "type",
- "role",
- "data-testid",
- "data-test-id",
- "data-cy",
- "data-qa",
- "href",
- "for",
- "alt",
- "title",
- "action",
- "method",
- "autocomplete",
- "inputmode",
- "pattern",
- "maxlength",
- "aria-describedby",
- "aria-labelledby",
- "aria-haspopup",
- "value", # useful for pre-selected state
- }
-)
-
-
-def _build_action_summary(a: Action) -> dict:
- """Build a rich action summary dict for the script reviewer.
-
- Includes a computed CSS selector suggestion so the reviewer can write
- reliable selectors without guessing from sparse attributes.
- """
- elem = a.skyvern_element_data or {}
- attrs = elem.get("attributes") or {}
-
- # Broad attribute set for the reviewer (filtered to safe, useful attrs)
- useful_attrs = {k: v for k, v in attrs.items() if k in _REVIEWER_SAFE_ATTRS and v}
-
- return {
- "action_type": a.action_type,
- "intention": a.intention,
- "reasoning": a.reasoning,
- "status": a.status,
- "field": (a.input_or_select_context.field if a.input_or_select_context else None),
- # Legacy: 6 core attributes (kept for backward compat with older templates)
- "element_attributes": (
- {k: v for k, v in attrs.items() if k in ("name", "id", "placeholder", "aria-label", "type", "role") and v}
- if attrs
- else None
- ),
- # New: element context for better selector generation
- "element_tag": elem.get("tagName"),
- "element_text": (elem.get("text") or "")[:100] or None,
- "all_attributes": useful_attrs or None,
- "css_suggestion": compute_stable_selector(elem),
- }
-
-
class WorkflowService:
@staticmethod
def _determine_cache_invalidation(
@@ -2147,7 +2083,7 @@ class WorkflowService:
task_id=wrb.task_id,
organization_id=organization_id,
)
- agent_actions_summary["actions"] = [_build_action_summary(a) for a in actions[:20]]
+ agent_actions_summary["actions"] = [build_action_summary(a) for a in actions[:20]]
except Exception:
LOG.debug(
"Could not fetch rich actions for fallback episode",
diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.py
index 7d7493c00..7a4f1ce5f 100644
--- a/skyvern/services/script_service.py
+++ b/skyvern/services/script_service.py
@@ -78,6 +78,7 @@ from skyvern.schemas.scripts import (
)
from skyvern.schemas.steps import AgentStepOutput
from skyvern.schemas.workflows import BlockResult, BlockStatus, BlockType, FileStorageType, FileType
+from skyvern.utils.css_selector import build_action_summary
from skyvern.webeye.actions.action_types import ActionType
from skyvern.webeye.actions.actions import Action, DecisiveAction
from skyvern.webeye.scraper.scraped_page import ElementTreeFormat
@@ -1351,24 +1352,12 @@ async def _fallback_to_ai_run(
agent_actions_summary["form_fields"] = form_fields_snapshot
if not fallback_succeeded and task.failure_reason:
agent_actions_summary["failure_reason"] = str(task.failure_reason)[:2000]
- # Fetch actions from the AI step.
- # NOTE: service.py uses _build_action_summary() which includes element
- # attributes and CSS selectors. We can't import it here (circular import:
- # service.py imports script_service). Core fields suffice for now.
try:
actions = await app.DATABASE.get_task_actions(
task_id=task_id,
organization_id=organization_id,
)
- agent_actions_summary["actions"] = [
- {
- "action_type": a.action_type,
- "intention": a.intention,
- "reasoning": a.reasoning,
- "status": a.status,
- }
- for a in actions[:20]
- ]
+ agent_actions_summary["actions"] = [build_action_summary(a) for a in actions[:20]]
except Exception:
LOG.debug("Could not fetch actions for fallback episode", exc_info=True)
diff --git a/skyvern/utils/css_selector.py b/skyvern/utils/css_selector.py
index be5eac8d0..a6ce7cda5 100644
--- a/skyvern/utils/css_selector.py
+++ b/skyvern/utils/css_selector.py
@@ -9,6 +9,10 @@ targets the same element without relying on ephemeral unique_ids.
from __future__ import annotations
import re
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from skyvern.webeye.actions.actions import Action
def compute_stable_selector(element_data: dict | None) -> str | None:
@@ -95,3 +99,72 @@ def _css_escape(s: str) -> str:
def _css_escape_attr(s: str) -> str:
"""Escape a string for use inside a CSS attribute value."""
return s.replace("\\", "\\\\").replace('"', '\\"')
+
+
+# ---------------------------------------------------------------------------
+# Action summary builder (shared between workflow service & script service)
+# ---------------------------------------------------------------------------
+
+# Attributes safe to pass to the script reviewer (excludes noisy/dynamic attrs)
+REVIEWER_SAFE_ATTRS = frozenset(
+ {
+ "name",
+ "id",
+ "placeholder",
+ "aria-label",
+ "type",
+ "role",
+ "data-testid",
+ "data-test-id",
+ "data-cy",
+ "data-qa",
+ "href",
+ "for",
+ "alt",
+ "title",
+ "action",
+ "method",
+ "autocomplete",
+ "inputmode",
+ "pattern",
+ "maxlength",
+ "aria-describedby",
+ "aria-labelledby",
+ "aria-haspopup",
+ "value", # useful for pre-selected state
+ }
+)
+
+
+def build_action_summary(action: Action) -> dict:
+ """Build a rich action summary dict for the script reviewer.
+
+ Includes a computed CSS selector suggestion so the reviewer can write
+ reliable selectors without guessing from sparse attributes.
+
+ Kept in this module (rather than workflow/service.py) so both the workflow
+ service and script_service can use it without circular imports.
+ """
+ elem = action.skyvern_element_data or {}
+ attrs = elem.get("attributes") or {}
+
+ useful_attrs = {k: v for k, v in attrs.items() if k in REVIEWER_SAFE_ATTRS and v}
+
+ return {
+ "action_type": action.action_type,
+ "intention": action.intention,
+ "reasoning": action.reasoning,
+ "status": action.status,
+ "field": (action.input_or_select_context.field if action.input_or_select_context else None),
+ # Legacy: 6 core attributes (kept for backward compat with older templates)
+ "element_attributes": (
+ {k: v for k, v in attrs.items() if k in ("name", "id", "placeholder", "aria-label", "type", "role") and v}
+ if attrs
+ else None
+ ),
+ # Element context for better selector generation
+ "element_tag": elem.get("tagName"),
+ "element_text": (elem.get("text") or "")[:100] or None,
+ "all_attributes": useful_attrs or None,
+ "css_suggestion": compute_stable_selector(elem),
+ }