Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
post action screenshot works for script run (#3235)
Shuchang Zhengb011842a71ec
diff --git a/skyvern/core/script_generations/skyvern_page.py b/skyvern/core/script_generations/skyvern_page.pyindex 1f1af0e26..89ed07834 100644--- a/skyvern/core/script_generations/skyvern_page.py+++ b/skyvern/core/script_generations/skyvern_page.py@@ -10,6 +10,7 @@ from typing import Any, Callable, Literalfrom playwright.async_api import Pagefrom skyvern.config import settings+from skyvern.exceptions import WorkflowRunNotFoundfrom skyvern.forge import appfrom skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.api.files import download_file@@ -17,7 +18,8 @@ from skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.webeye.actions import handler_utilsfrom skyvern.webeye.actions.action_types import ActionType-from skyvern.webeye.actions.actions import Action+from skyvern.webeye.actions.actions import Action, ActionStatus+from skyvern.webeye.browser_factory import BrowserStatefrom skyvern.webeye.scraper.scraper import ScrapedPage, scrape_website@@ -63,10 +65,43 @@ class SkyvernPage:self.page = pageself._record = recorder or (lambda ac: None)+ @classmethod+ async def _get_or_create_browser_state(cls) -> BrowserState:+ context = skyvern_context.current()+ if context and context.workflow_run_id and context.organization_id:+ workflow_run = await app.DATABASE.get_workflow_run(+ workflow_run_id=context.workflow_run_id, organization_id=context.organization_id+ )+ if workflow_run:+ browser_state = await app.BROWSER_MANAGER.get_or_create_for_workflow_run(+ workflow_run=workflow_run, browser_session_id=None+ )+ else:+ raise WorkflowRunNotFound(workflow_run_id=context.workflow_run_id)+ else:+ browser_state = await app.BROWSER_MANAGER.get_or_create_for_script()+ return browser_state++ @classmethod+ async def _get_browser_state(cls) -> BrowserState | None:+ context = skyvern_context.current()+ if context and context.workflow_run_id and context.organization_id:+ workflow_run = await app.DATABASE.get_workflow_run(+ workflow_run_id=context.workflow_run_id, organization_id=context.organization_id+ )+ if workflow_run:+ browser_state = app.BROWSER_MANAGER.get_for_workflow_run(workflow_run_id=context.workflow_run_id)+ else:+ raise WorkflowRunNotFound(workflow_run_id=context.workflow_run_id)+ else:+ browser_state = app.BROWSER_MANAGER.get_for_script()+ return browser_state+@classmethodasync def create(cls) -> SkyvernPage:# initialize browser state- browser_state = await app.BROWSER_MANAGER.get_or_create_for_script()+ # TODO: add workflow_run_id or eventually script_id/script_run_id+ browser_state = await cls._get_or_create_browser_state()scraped_page = await scrape_website(browser_state=browser_state,url="",@@ -102,10 +137,7 @@ class SkyvernPage:meta = ActionMetadata(intention, data)call = ActionCall(action, args, kwargs, meta)- # Auto-create action before execution- action_record = await skyvern_page._create_action_before_execution(- action_type=action, intention=intention, data=data- )+ action_status = ActionStatus.completedtry:call.result = await fn(@@ -117,16 +149,23 @@ class SkyvernPage:return call.resultexcept Exception as e:call.error = e-+ action_status = ActionStatus.failed# Note: Action status would be updated to failed here if update method existed# LLM fallback hook could go here ...raisefinally:skyvern_page._record(call)+ # Auto-create action before execution+ await skyvern_page._create_action_before_execution(+ action_type=action,+ intention=intention,+ status=action_status,+ data=data,+ )# Auto-create screenshot artifact after execution- await skyvern_page._create_screenshot_after_execution(action_record)+ await skyvern_page._create_screenshot_after_execution()return wrapper@@ -136,7 +175,11 @@ class SkyvernPage:await self.page.goto(url)async def _create_action_before_execution(- self, action_type: ActionType, intention: str = "", data: str | dict[str, Any] = ""+ self,+ action_type: ActionType,+ intention: str = "",+ status: ActionStatus = ActionStatus.pending,+ data: str | dict[str, Any] = "",) -> Action | None:"""Create an action record in the database before execution if task_id and step_id are available."""try:@@ -144,9 +187,10 @@ class SkyvernPage:if not context or not context.task_id or not context.step_id:return None- # Create action record+ # Create action record. TODO: store more action fieldsaction = Action(action_type=action_type,+ status=status,organization_id=context.organization_id,workflow_run_id=context.workflow_run_id,task_id=context.task_id,@@ -164,7 +208,8 @@ class SkyvernPage:# If action creation fails, don't block the actual action executionreturn None- async def _create_screenshot_after_execution(self, action_record: Action | None = None) -> None:+ @classmethod+ async def _create_screenshot_after_execution(cls) -> None:"""Create a screenshot artifact after action execution if task_id and step_id are available."""try:context = skyvern_context.ensure_context()@@ -172,17 +217,11 @@ class SkyvernPage:return# Get browser state and take screenshot- browser_state = await app.BROWSER_MANAGER.get_for_script()+ browser_state = await cls._get_browser_state()if not browser_state:return- screenshot = await browser_state.take_fullpage_screenshot(- use_playwright_fullpage=app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached(- "ENABLE_PLAYWRIGHT_FULLPAGE",- context.workflow_run_id or context.task_id,- properties={"organization_id": context.organization_id},- )- )+ screenshot = await browser_state.take_post_action_screenshot(scrolling_number=0)if screenshot:# Create a minimal Step object for artifact creationdiff --git a/skyvern/webeye/browser_manager.py b/skyvern/webeye/browser_manager.pyindex cb515808e..eff7c703b 100644--- a/skyvern/webeye/browser_manager.py+++ b/skyvern/webeye/browser_manager.py@@ -385,7 +385,7 @@ class BrowserManager:script_id: str | None = None,browser_session_id: str | None = None,) -> BrowserState:- browser_state = await self.get_for_script(script_id=script_id)+ browser_state = self.get_for_script(script_id=script_id)if browser_state:return browser_state@@ -422,7 +422,7 @@ class BrowserManager:return browser_state- async def get_for_script(self, script_id: str | None = None) -> BrowserState | None:+ def get_for_script(self, script_id: str | None = None) -> BrowserState | None:if script_id and script_id in self.pages:return self.pages[script_id]return None