Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
add action result, including downloaded files to step.output for both agent and caching runs (#4477)
Shuchang Zhengbd041ed52f11
diff --git a/skyvern/core/script_generations/script_skyvern_page.py b/skyvern/core/script_generations/script_skyvern_page.pyindex d545cdc84..9e4d67784 100644--- a/skyvern/core/script_generations/script_skyvern_page.py+++ b/skyvern/core/script_generations/script_skyvern_page.py@@ -1,20 +1,30 @@from __future__ import annotationsimport asyncio+import os+from pathlib import Pathfrom typing import Any, Callableimport structlogfrom playwright.async_api import Pagefrom skyvern.config import settings+from skyvern.constants import BROWSER_DOWNLOAD_TIMEOUTfrom skyvern.core.script_generations.real_skyvern_page_ai import RealSkyvernPageAi, render_templatefrom skyvern.core.script_generations.skyvern_page import ActionCall, ActionMetadata, RunContext, SkyvernPagefrom skyvern.core.script_generations.skyvern_page_ai import SkyvernPageAi+from skyvern.errors.errors import UserDefinedErrorfrom skyvern.exceptions import ScriptTerminationException, WorkflowRunNotFoundfrom skyvern.forge import appfrom skyvern.forge.prompts import prompt_engine+from skyvern.forge.sdk.api.files import (+ check_downloading_files_and_wait_for_download_to_complete,+ get_path_for_workflow_download_directory,+ list_files_in_directory,+)from skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.core import skyvern_context+from skyvern.schemas.steps import AgentStepOutputfrom skyvern.services.otp_service import poll_otp_valuefrom skyvern.utils.url_validators import prepend_scheme_and_validate_urlfrom skyvern.webeye.actions.action_types import ActionType@@ -22,6 +32,7 @@ from skyvern.webeye.actions.actions import (Action,ActionStatus,CompleteAction,+ DecisiveAction,ExtractAction,SelectOption,SolveCaptchaAction,@@ -32,6 +43,7 @@ from skyvern.webeye.actions.handler import (get_actual_value_of_parameter_if_secret,handle_complete_action,)+from skyvern.webeye.actions.responses import ActionFailure, ActionResult, ActionSuccessfrom skyvern.webeye.browser_state import BrowserStatefrom skyvern.webeye.scraper.scraped_page import ScrapedPage@@ -122,6 +134,29 @@ class ScriptSkyvernPage(SkyvernPage):support_empty_page=True,)+ async def _ensure_download_to_complete(+ self,+ download_dir: Path,+ browser_session_id: str | None = None,+ ) -> None:+ context = skyvern_context.current()+ if not context or not context.organization_id:+ return+ if not download_dir.exists():+ return+ organization_id = context.organization_id+ download_timeout = BROWSER_DOWNLOAD_TIMEOUT+ if context.task_id:+ task = await app.DATABASE.get_task(context.task_id, organization_id=organization_id)+ if task and task.download_timeout:+ download_timeout = task.download_timeout+ await check_downloading_files_and_wait_for_download_to_complete(+ download_dir=download_dir,+ organization_id=organization_id,+ browser_session_id=browser_session_id,+ timeout=download_timeout,+ )+async def _decorate_call(self,fn: Callable,@@ -176,6 +211,28 @@ class ScriptSkyvernPage(SkyvernPage):else:print()+ # Download detection for click actions+ download_triggered: bool | None = None+ downloaded_files: list[str] | None = None+ files_before: list[str] = []+ download_dir: Path | None = None++ # Capture files before click action for download detection+ if action == ActionType.CLICK and context and context.workflow_run_id:+ try:+ download_dir = get_path_for_workflow_download_directory(context.workflow_run_id)+ if download_dir.exists():+ files_before = list_files_in_directory(download_dir)+ if context.browser_session_id and context.organization_id:+ browser_session_downloaded_files = await app.STORAGE.list_downloaded_files_in_browser_session(+ organization_id=context.organization_id,+ browser_session_id=context.browser_session_id,+ )+ files_before = files_before + browser_session_downloaded_files++ except Exception:+ pass # Don't block action execution if file listing fails+try:call.result = await fn(self, *args, **kwargs)@@ -198,23 +255,58 @@ class ScriptSkyvernPage(SkyvernPage):# LLM fallback hook could go here ...raisefinally:+ # Add a small buffer between cached actions to give slow pages time to settle+ if settings.CACHED_ACTION_DELAY_SECONDS > 0:+ await asyncio.sleep(settings.CACHED_ACTION_DELAY_SECONDS)++ # Check for downloaded files after click action+ if action == ActionType.CLICK and context and context.workflow_run_id and download_dir:+ try:+ if download_dir.exists():+ await self._ensure_download_to_complete(+ download_dir=download_dir,+ browser_session_id=context.browser_session_id,+ )+ files_after = list_files_in_directory(download_dir)+ if context.browser_session_id and context.organization_id:+ browser_session_downloaded_files = (+ await app.STORAGE.list_downloaded_files_in_browser_session(+ organization_id=context.organization_id,+ browser_session_id=context.browser_session_id,+ )+ )+ files_after = files_after + browser_session_downloaded_files++ new_file_paths = set(files_after) - set(files_before)+ if new_file_paths:+ download_triggered = True+ downloaded_files = [os.path.basename(fp) for fp in new_file_paths]+ LOG.info(+ "Script click action detected download",+ downloaded_files=downloaded_files,+ workflow_run_id=context.workflow_run_id,+ )+ else:+ download_triggered = False+ except Exception:+ pass # Don't block if download detection fails+self._record(call)- # Auto-create action after execution- await self._create_action_after_execution(+ # Auto-create action after execution and store result+ await self._create_action_and_result_after_execution(action_type=action,intention=prompt,status=action_status,kwargs=kwargs,call_result=call.result,+ call_error=call.error,+ download_triggered=download_triggered,+ downloaded_files=downloaded_files,)# Auto-create screenshot artifact after executionawait self._create_screenshot_after_execution()- # Add a small buffer between cached actions to give slow pages time to settle- if settings.CACHED_ACTION_DELAY_SECONDS > 0:- await asyncio.sleep(settings.CACHED_ACTION_DELAY_SECONDS)-async def _update_action_reasoning(self,action_id: str,@@ -265,20 +357,27 @@ class ScriptSkyvernPage(SkyvernPage):)return reasoning- async def _create_action_after_execution(+ async def _create_action_and_result_after_execution(self,action_type: ActionType,intention: str = "",status: ActionStatus = ActionStatus.pending,kwargs: dict[str, Any] | None = None,call_result: Any | None = None,- ) -> Action | None:- """Create an action record in the database before execution if task_id and step_id are available."""+ call_error: Exception | None = None,+ download_triggered: bool | None = None,+ downloaded_files: list[str] | None = None,+ ) -> tuple[Action | None, list[ActionResult]]:+ """Create an action record and result in the database after execution if task_id and step_id are available.++ Returns a tuple of (Action, list[ActionResult]) similar to how the agent stores actions and results.+ """+ results: list[ActionResult] = []try:context = skyvern_context.current()if not context or not context.task_id or not context.step_id:- return None+ return None, results# Create action record. TODO: store more action fieldskwargs = kwargs or {}@@ -320,6 +419,9 @@ class ScriptSkyvernPage(SkyvernPage):file_url=file_url,response=response,xpath=xpath,+ download=download_triggered,+ download_triggered=download_triggered,+ downloaded_files=downloaded_files,created_by="script",)data_extraction_goal = None@@ -363,11 +465,32 @@ class ScriptSkyvernPage(SkyvernPage):context.action_order += 1- return created_action+ # Create ActionResult based on success/failure+ if call_error:+ result = ActionFailure(exception=call_error, download_triggered=download_triggered)+ else:+ # For extract actions, include the extracted data in the result+ result_data = None+ if action_type == ActionType.EXTRACT and call_result:+ result_data = call_result+ result = ActionSuccess(+ data=result_data,+ download_triggered=download_triggered,+ downloaded_files=downloaded_files,+ )++ results = [result]++ # Store action and results in RunContext for step output+ run_context = script_run_context_manager.get_run_context()+ if run_context:+ run_context.actions_and_results.append((created_action, results))++ return created_action, resultsexcept Exception:# If action creation fails, don't block the actual action execution- return None+ return None, results@classmethodasync def _create_screenshot_after_execution(cls) -> None:@@ -492,6 +615,15 @@ class ScriptSkyvernPage(SkyvernPage):task = await app.DATABASE.get_task(context.task_id, context.organization_id)step = await app.DATABASE.get_step(context.step_id, context.organization_id)if task and step:+ # CRITICAL: Update step.output with actions_and_results BEFORE validation+ # This ensures complete_verify() can access action history (including download info)+ # when checking if the goal was achieved+ await self._update_step_output_before_complete(context)+ # Refresh step to get updated output for validation+ step = await app.DATABASE.get_step(context.step_id, context.organization_id)+ if not step:+ return+action = CompleteAction(organization_id=context.organization_id,task_id=context.task_id,@@ -504,6 +636,48 @@ class ScriptSkyvernPage(SkyvernPage):if result and result[-1].success is False:raise ScriptTerminationException(result[-1].exception_message)+ async def _update_step_output_before_complete(self, context: skyvern_context.SkyvernContext) -> None:+ """Update step.output with actions_and_results before complete validation.++ This is critical for cached runs because complete_verify() reads action history+ from step.output.actions_and_results to check if goals were achieved (e.g., file downloads).+ Without this, the validation has no visibility into what actions were performed.+ """++ # Validate required context fields+ if not context.step_id or not context.task_id or not context.organization_id:+ return++ run_context = script_run_context_manager.get_run_context()+ if not run_context or not run_context.actions_and_results:+ return++ # Extract errors from DecisiveActions (similar to agent flow)+ errors: list[UserDefinedError] = []+ for action, _ in run_context.actions_and_results:+ if isinstance(action, DecisiveAction):+ errors.extend(action.errors)++ # Create AgentStepOutput similar to how agent does it+ step_output = AgentStepOutput(+ actions_and_results=run_context.actions_and_results,+ action_results=[result for _, results in run_context.actions_and_results for result in results],+ errors=errors,+ )++ await app.DATABASE.update_step(+ step_id=context.step_id,+ task_id=context.task_id,+ organization_id=context.organization_id,+ output=step_output,+ )+ LOG.info(+ "Updated step output with cached actions before complete validation",+ step_id=context.step_id,+ task_id=context.task_id,+ num_actions=len(run_context.actions_and_results),+ )+class ScriptRunContextManager:"""diff --git a/skyvern/core/script_generations/skyvern_page.py b/skyvern/core/script_generations/skyvern_page.pyindex 5938647df..56def8057 100644--- a/skyvern/core/script_generations/skyvern_page.py+++ b/skyvern/core/script_generations/skyvern_page.py@@ -3,7 +3,7 @@ from __future__ import annotationsimport asyncioimport copyfrom dataclasses import dataclass-from typing import Any, Callable, Literal, overload+from typing import TYPE_CHECKING, Any, Callable, Literal, overloadimport structlogfrom playwright.async_api import Locator, Page@@ -16,6 +16,10 @@ from skyvern.library.ai_locator import AILocatorfrom skyvern.webeye.actions import handler_utilsfrom skyvern.webeye.actions.action_types import ActionType+if TYPE_CHECKING:+ from skyvern.webeye.actions.actions import Action+ from skyvern.webeye.actions.responses import ActionResult+LOG = structlog.get_logger()@@ -1007,3 +1011,5 @@ class RunContext:self.parameters[key] = valueself.page = pageself.trace: list[ActionCall] = []+ # Store actions and results for step output (similar to agent flow)+ self.actions_and_results: list[tuple[Action, list[ActionResult]]] = []diff --git a/skyvern/forge/sdk/db/agent_db.py b/skyvern/forge/sdk/db/agent_db.pyindex 369ebce59..4e4fd6840 100644--- a/skyvern/forge/sdk/db/agent_db.py+++ b/skyvern/forge/sdk/db/agent_db.py@@ -3859,7 +3859,7 @@ class AgentDB(BaseAlchemyDB):session.add(new_action)await session.commit()await session.refresh(new_action)- return Action.model_validate(new_action)+ return hydrate_action(new_action)async def update_action_screenshot_artifact_id(self, *, organization_id: str, action_id: str, screenshot_artifact_id: strdiff --git a/skyvern/services/action_service.py b/skyvern/services/action_service.pyindex d529f15cc..d4b2fcd9d 100644--- a/skyvern/services/action_service.py+++ b/skyvern/services/action_service.py@@ -34,7 +34,14 @@ async def get_action_history({"action": action.model_dump(exclude_none=True,- include={"action_type", "element_id", "status", "reasoning", "option", "download"},+ include={+ "action_type",+ "element_id",+ "status",+ "reasoning",+ "option",+ "download",+ },),# use the last result of the action, because some actions(like chain_click)# might have multiple results. Only the last one can represent the real result,diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.pyindex 5d0d3a9bd..0ce4cd4d7 100644--- a/skyvern/services/script_service.py+++ b/skyvern/services/script_service.py@@ -60,9 +60,10 @@ from skyvern.schemas.scripts import (ScriptFileCreate,ScriptStatus,)+from skyvern.schemas.steps import AgentStepOutputfrom skyvern.schemas.workflows import BlockResult, BlockStatus, BlockType, FileStorageType, FileTypefrom skyvern.webeye.actions.action_types import ActionType-from skyvern.webeye.actions.actions import Action+from skyvern.webeye.actions.actions import Action, DecisiveActionfrom skyvern.webeye.scraper.scraped_page import ElementTreeFormatLOG = structlog.get_logger()@@ -455,6 +456,8 @@ async def _create_workflow_block_run_and_task(organization_id=organization_id,workflow_run_id=workflow_run_id,model=model,+ # always use the action history for validation in caching/script run+ include_action_history_in_verification=True,)task_id = task.task_id@@ -593,12 +596,30 @@ async def _update_workflow_block(final_output = outputif task_id:if step_id:+ # Build step output from script actions similar to agent flow+ step_output = None+ run_context = script_run_context_manager.get_run_context()+ if run_context and run_context.actions_and_results:+ # Extract errors from DecisiveActions (similar to agent flow)+ errors: list[UserDefinedError] = []+ for action, _ in run_context.actions_and_results:+ if isinstance(action, DecisiveAction):+ errors.extend(action.errors)++ # Create AgentStepOutput similar to how agent does it+ step_output = AgentStepOutput(+ actions_and_results=run_context.actions_and_results,+ action_results=[result for _, results in run_context.actions_and_results for result in results],+ errors=errors,+ )+await app.DATABASE.update_step(step_id=step_id,task_id=task_id,organization_id=context.organization_id,status=step_status,is_last=is_last,+ output=step_output,)updated_task = await app.DATABASE.update_task(task_id=task_id,diff --git a/skyvern/webeye/actions/actions.py b/skyvern/webeye/actions/actions.pyindex c3e110f8b..477c4d78b 100644--- a/skyvern/webeye/actions/actions.py+++ b/skyvern/webeye/actions/actions.py@@ -125,6 +125,8 @@ class Action(BaseModel):file_name: str | None = Nonefile_url: str | None = Nonedownload: bool | None = None+ download_triggered: bool | None = None # Whether a download was triggered by this action+ downloaded_files: list[str] | None = None # List of file names downloaded by this actionis_upload_file_tag: bool | None = Nonetext: str | None = Noneinput_or_select_context: InputOrSelectContext | None = Nonediff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 2fd7891ee..3ab5a96bd 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -481,8 +481,22 @@ class ActionHandler:if not download_triggered:results[-1].download_triggered = False+ action.download_triggered = Falsereturn resultsresults[-1].download_triggered = True+ action.download_triggered = True++ # Calculate newly downloaded file names+ new_file_paths = set(list_files_after) - set(list_files_before)+ downloaded_file_names = [os.path.basename(fp) for fp in new_file_paths]+ if downloaded_file_names:+ results[-1].downloaded_files = downloaded_file_names+ action.downloaded_files = downloaded_file_names+ LOG.info(+ "Downloaded files captured",+ downloaded_files=downloaded_file_names,+ workflow_run_id=task.workflow_run_id,+ )await check_downloading_files_and_wait_for_download_to_complete(download_dir=download_dir,diff --git a/skyvern/webeye/actions/responses.py b/skyvern/webeye/actions/responses.pyindex 1fc1c445b..07f473ebb 100644--- a/skyvern/webeye/actions/responses.py+++ b/skyvern/webeye/actions/responses.py@@ -14,6 +14,7 @@ class ActionResult(BaseModel):step_retry_number: int | None = Nonestep_order: int | None = Nonedownload_triggered: bool | None = None+ downloaded_files: list[str] | None = None # Actual file names that were downloaded# None is used for old data so that we can differentiate between old and new data which only has booleaninteracted_with_sibling: bool | None = Noneinteracted_with_parent: bool | None = None@@ -33,6 +34,8 @@ class ActionResult(BaseModel):results.append(f"step_retry_number={self.step_retry_number}")if self.download_triggered is not None:results.append(f"download_triggered={self.download_triggered}")+ if self.downloaded_files is not None:+ results.append(f"downloaded_files={self.downloaded_files}")if self.interacted_with_sibling is not None:results.append(f"interacted_with_sibling={self.interacted_with_sibling}")if self.interacted_with_parent is not None:@@ -51,6 +54,7 @@ class ActionSuccess(ActionResult):self,data: dict[str, Any] | list | str | None = None,download_triggered: bool | None = None,+ downloaded_files: list[str] | None = None,interacted_with_sibling: bool = False,interacted_with_parent: bool = False,):@@ -58,6 +62,7 @@ class ActionSuccess(ActionResult):success=True,data=data,download_triggered=download_triggered,+ downloaded_files=downloaded_files,interacted_with_sibling=interacted_with_sibling,interacted_with_parent=interacted_with_parent,)