Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-8517): rich action data for script reviewer from AI fallback episodes (#5218)
pedrohsdb82628cdd2f66
diff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.pyindex 79d6d753d..55c3783eb 100644--- a/skyvern/forge/sdk/routes/agent_protocol.py+++ b/skyvern/forge/sdk/routes/agent_protocol.py@@ -1082,6 +1082,7 @@ async def delete_workflow("/folders",response_model=Folder,tags=["Workflow Folders"],+ include_in_schema=False,openapi_extra={"x-fern-sdk-method-name": "create_folder",},@@ -1124,6 +1125,7 @@ async def create_folder("/folders/{folder_id}",response_model=Folder,tags=["Workflow Folders"],+ include_in_schema=False,openapi_extra={"x-fern-sdk-method-name": "get_folder",},@@ -1168,6 +1170,7 @@ async def get_folder("/folders",response_model=list[Folder],tags=["Workflow Folders"],+ include_in_schema=False,openapi_extra={"x-fern-sdk-method-name": "get_folders",},@@ -1225,6 +1228,7 @@ async def get_folders("/folders/{folder_id}",response_model=Folder,tags=["Workflow Folders"],+ include_in_schema=False,openapi_extra={"x-fern-sdk-method-name": "update_folder",},@@ -1271,6 +1275,7 @@ async def update_folder(@base_router.delete("/folders/{folder_id}",tags=["Workflow Folders"],+ include_in_schema=False,openapi_extra={"x-fern-sdk-method-name": "delete_folder",},@@ -1307,6 +1312,7 @@ async def delete_folder("/workflows/{workflow_permanent_id}/folder",response_model=Workflow,tags=["Workflow Folders"],+ include_in_schema=False,openapi_extra={"x-fern-sdk-method-name": "update_workflow_folder",},@@ -1397,6 +1403,7 @@ async def convert_curl_to_http("/artifacts/{artifact_id}",tags=["Artifacts"],response_model=Artifact,+ include_in_schema=False,openapi_extra={"x-fern-sdk-method-name": "get_artifact",},diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 4ef40fc73..1c2f13434 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -121,8 +121,7 @@ from skyvern.schemas.workflows import (WorkflowStatus,)from skyvern.services import script_service, workflow_script_service-from skyvern.utils.css_selector import compute_stable_selector-from skyvern.webeye.actions.actions import Action+from skyvern.utils.css_selector import build_action_summary # shared with script_servicefrom skyvern.webeye.browser_state import BrowserStateLOG = structlog.get_logger()@@ -290,69 +289,6 @@ def _get_workflow_definition_core_data(workflow_definition: WorkflowDefinition)return workflow_dict-# Attributes safe to pass to the script reviewer (excludes noisy/dynamic attrs)-_REVIEWER_SAFE_ATTRS = frozenset(- {- "name",- "id",- "placeholder",- "aria-label",- "type",- "role",- "data-testid",- "data-test-id",- "data-cy",- "data-qa",- "href",- "for",- "alt",- "title",- "action",- "method",- "autocomplete",- "inputmode",- "pattern",- "maxlength",- "aria-describedby",- "aria-labelledby",- "aria-haspopup",- "value", # useful for pre-selected state- }-)---def _build_action_summary(a: Action) -> dict:- """Build a rich action summary dict for the script reviewer.-- Includes a computed CSS selector suggestion so the reviewer can write- reliable selectors without guessing from sparse attributes.- """- elem = a.skyvern_element_data or {}- attrs = elem.get("attributes") or {}-- # Broad attribute set for the reviewer (filtered to safe, useful attrs)- useful_attrs = {k: v for k, v in attrs.items() if k in _REVIEWER_SAFE_ATTRS and v}-- return {- "action_type": a.action_type,- "intention": a.intention,- "reasoning": a.reasoning,- "status": a.status,- "field": (a.input_or_select_context.field if a.input_or_select_context else None),- # Legacy: 6 core attributes (kept for backward compat with older templates)- "element_attributes": (- {k: v for k, v in attrs.items() if k in ("name", "id", "placeholder", "aria-label", "type", "role") and v}- if attrs- else None- ),- # New: element context for better selector generation- "element_tag": elem.get("tagName"),- "element_text": (elem.get("text") or "")[:100] or None,- "all_attributes": useful_attrs or None,- "css_suggestion": compute_stable_selector(elem),- }--class WorkflowService:@staticmethoddef _determine_cache_invalidation(@@ -2147,7 +2083,7 @@ class WorkflowService:task_id=wrb.task_id,organization_id=organization_id,)- agent_actions_summary["actions"] = [_build_action_summary(a) for a in actions[:20]]+ agent_actions_summary["actions"] = [build_action_summary(a) for a in actions[:20]]except Exception:LOG.debug("Could not fetch rich actions for fallback episode",diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.pyindex 7d7493c00..7a4f1ce5f 100644--- a/skyvern/services/script_service.py+++ b/skyvern/services/script_service.py@@ -78,6 +78,7 @@ from skyvern.schemas.scripts import ()from skyvern.schemas.steps import AgentStepOutputfrom skyvern.schemas.workflows import BlockResult, BlockStatus, BlockType, FileStorageType, FileType+from skyvern.utils.css_selector import build_action_summaryfrom skyvern.webeye.actions.action_types import ActionTypefrom skyvern.webeye.actions.actions import Action, DecisiveActionfrom skyvern.webeye.scraper.scraped_page import ElementTreeFormat@@ -1351,24 +1352,12 @@ async def _fallback_to_ai_run(agent_actions_summary["form_fields"] = form_fields_snapshotif not fallback_succeeded and task.failure_reason:agent_actions_summary["failure_reason"] = str(task.failure_reason)[:2000]- # Fetch actions from the AI step.- # NOTE: service.py uses _build_action_summary() which includes element- # attributes and CSS selectors. We can't import it here (circular import:- # service.py imports script_service). Core fields suffice for now.try:actions = await app.DATABASE.get_task_actions(task_id=task_id,organization_id=organization_id,)- agent_actions_summary["actions"] = [- {- "action_type": a.action_type,- "intention": a.intention,- "reasoning": a.reasoning,- "status": a.status,- }- for a in actions[:20]- ]+ agent_actions_summary["actions"] = [build_action_summary(a) for a in actions[:20]]except Exception:LOG.debug("Could not fetch actions for fallback episode", exc_info=True)diff --git a/skyvern/utils/css_selector.py b/skyvern/utils/css_selector.pyindex be5eac8d0..a6ce7cda5 100644--- a/skyvern/utils/css_selector.py+++ b/skyvern/utils/css_selector.py@@ -9,6 +9,10 @@ targets the same element without relying on ephemeral unique_ids.from __future__ import annotationsimport re+from typing import TYPE_CHECKING++if TYPE_CHECKING:+ from skyvern.webeye.actions.actions import Actiondef compute_stable_selector(element_data: dict | None) -> str | None:@@ -95,3 +99,72 @@ def _css_escape(s: str) -> str:def _css_escape_attr(s: str) -> str:"""Escape a string for use inside a CSS attribute value."""return s.replace("\\", "\\\\").replace('"', '\\"')+++# ---------------------------------------------------------------------------+# Action summary builder (shared between workflow service & script service)+# ---------------------------------------------------------------------------++# Attributes safe to pass to the script reviewer (excludes noisy/dynamic attrs)+REVIEWER_SAFE_ATTRS = frozenset(+ {+ "name",+ "id",+ "placeholder",+ "aria-label",+ "type",+ "role",+ "data-testid",+ "data-test-id",+ "data-cy",+ "data-qa",+ "href",+ "for",+ "alt",+ "title",+ "action",+ "method",+ "autocomplete",+ "inputmode",+ "pattern",+ "maxlength",+ "aria-describedby",+ "aria-labelledby",+ "aria-haspopup",+ "value", # useful for pre-selected state+ }+)+++def build_action_summary(action: Action) -> dict:+ """Build a rich action summary dict for the script reviewer.++ Includes a computed CSS selector suggestion so the reviewer can write+ reliable selectors without guessing from sparse attributes.++ Kept in this module (rather than workflow/service.py) so both the workflow+ service and script_service can use it without circular imports.+ """+ elem = action.skyvern_element_data or {}+ attrs = elem.get("attributes") or {}++ useful_attrs = {k: v for k, v in attrs.items() if k in REVIEWER_SAFE_ATTRS and v}++ return {+ "action_type": action.action_type,+ "intention": action.intention,+ "reasoning": action.reasoning,+ "status": action.status,+ "field": (action.input_or_select_context.field if action.input_or_select_context else None),+ # Legacy: 6 core attributes (kept for backward compat with older templates)+ "element_attributes": (+ {k: v for k, v in attrs.items() if k in ("name", "id", "placeholder", "aria-label", "type", "role") and v}+ if attrs+ else None+ ),+ # Element context for better selector generation+ "element_tag": elem.get("tagName"),+ "element_text": (elem.get("text") or "")[:100] or None,+ "all_attributes": useful_attrs or None,+ "css_suggestion": compute_stable_selector(elem),+ }