Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
integration with CUA (#2126)
Shuchang Zhengf883b91180af
diff --git a/skyvern/config.py b/skyvern/config.pyindex db1079d67..223ab1537 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -116,6 +116,7 @@ class Settings(BaseSettings):ENABLE_AZURE_O3_MINI: bool = FalseENABLE_BEDROCK: bool = FalseENABLE_GEMINI: bool = False+ ENABLE_AZURE_CUA: bool = False# OPENAIOPENAI_API_KEY: str | None = None# ANTHROPIC@@ -125,6 +126,10 @@ class Settings(BaseSettings):AZURE_API_KEY: str | None = NoneAZURE_API_BASE: str | None = NoneAZURE_API_VERSION: str | None = None+ AZURE_CUA_API_KEY: str | None = None+ AZURE_CUA_ENDPOINT: str | None = None+ AZURE_CUA_DEPLOYMENT: str | None = "computer-use-preview"+ AZURE_CUA_API_VERSION: str | None = "2025-03-01-preview"# AZURE GPT-4o miniAZURE_GPT4O_MINI_DEPLOYMENT: str | None = Nonediff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex d6e8e0665..a5bff0724 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -1,4 +1,5 @@import asyncio+import base64import jsonimport osimport random@@ -10,6 +11,7 @@ from typing import Any, Tupleimport httpximport structlog+from openai.types.responses.response import Response as OpenAIResponsefrom playwright._impl._errors import TargetClosedErrorfrom playwright.async_api import Page@@ -68,6 +70,7 @@ from skyvern.forge.sdk.schemas.tasks import Task, TaskRequest, TaskResponse, Tasfrom skyvern.forge.sdk.workflow.context_manager import WorkflowRunContextfrom skyvern.forge.sdk.workflow.models.block import ActionBlock, BaseTaskBlock, ValidationBlockfrom skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowRun, WorkflowRunStatus+from skyvern.schemas.runs import RunEngine, RunTypefrom skyvern.utils.prompt_engine import load_prompt_with_elementsfrom skyvern.webeye.actions.actions import (Action,@@ -85,7 +88,7 @@ from skyvern.webeye.actions.actions import (from skyvern.webeye.actions.caching import retrieve_action_planfrom skyvern.webeye.actions.handler import ActionHandler, poll_verification_codefrom skyvern.webeye.actions.models import AgentStepOutput, DetailedAgentStepOutput-from skyvern.webeye.actions.parse_actions import parse_actions+from skyvern.webeye.actions.parse_actions import parse_actions, parse_cua_actionsfrom skyvern.webeye.actions.responses import ActionResult, ActionSuccessfrom skyvern.webeye.browser_factory import BrowserStatefrom skyvern.webeye.scraper.scraper import ElementTreeFormat, ScrapedPage, scrape_website@@ -248,6 +251,8 @@ class ForgeAgent:task_block: BaseTaskBlock | None = None,browser_session_id: str | None = None,complete_verification: bool = True,+ engine: RunEngine = RunEngine.skyvern_v1,+ cua_response: OpenAIResponse | None = None,) -> Tuple[Step, DetailedAgentStepOutput | None, Step | None]:workflow_run: WorkflowRun | None = Noneif task.workflow_run_id:@@ -380,6 +385,8 @@ class ForgeAgent:organization=organization,task_block=task_block,complete_verification=complete_verification,+ engine=engine,+ cua_response=cua_response,)await app.AGENT_FUNCTION.post_step_execution(task, step)task = await self.update_task_errors_from_detailed_output(task, detailed_output)@@ -506,6 +513,10 @@ class ForgeAgent:step_status=step.status,)+ cua_response_param = detailed_output.cua_response if detailed_output else None+ if not cua_response_param and cua_response:+ cua_response_param = cua_response+if retry and next_step:return await self.execute_step(organization,@@ -516,6 +527,8 @@ class ForgeAgent:browser_session_id=browser_session_id,task_block=task_block,complete_verification=complete_verification,+ engine=engine,+ cua_response=cua_response_param,)elif settings.execute_all_steps() and next_step:return await self.execute_step(@@ -527,6 +540,8 @@ class ForgeAgent:browser_session_id=browser_session_id,task_block=task_block,complete_verification=complete_verification,+ engine=engine,+ cua_response=cua_response_param,)else:LOG.info(@@ -757,9 +772,11 @@ class ForgeAgent:task: Task,step: Step,browser_state: BrowserState,+ engine: RunEngine = RunEngine.skyvern_v1,organization: Organization | None = None,task_block: BaseTaskBlock | None = None,complete_verification: bool = True,+ cua_response: OpenAIResponse | None = None,) -> tuple[Step, DetailedAgentStepOutput]:detailed_agent_step_output = DetailedAgentStepOutput(scraped_page=None,@@ -768,6 +785,7 @@ class ForgeAgent:actions=None,action_results=None,actions_and_results=None,+ cua_response=None,)try:LOG.info(@@ -789,52 +807,62 @@ class ForgeAgent:task,step,browser_state,+ engine,)detailed_agent_step_output.scraped_page = scraped_pagedetailed_agent_step_output.extract_action_prompt = extract_action_promptjson_response = Noneactions: list[Action]- using_cached_action_plan = False- if not task.navigation_goal and not isinstance(task_block, ValidationBlock):- actions = [await self.create_extract_action(task, step, scraped_page)]- elif (- task_block- and task_block.cache_actions- and (actions := await retrieve_action_plan(task, step, scraped_page))- ):- using_cached_action_plan = True- else:- self.async_operation_pool.run_operation(task.task_id, AgentPhase.llm)- json_response = await app.LLM_API_HANDLER(- prompt=extract_action_prompt,- prompt_name="extract-actions",+ if engine == RunEngine.openai_cua:+ actions, new_cua_response = await self._generate_cua_actions(+ task=task,step=step,- screenshots=scraped_page.screenshots,+ scraped_page=scraped_page,+ previous_response=cua_response,)- try:- json_response = await self.handle_potential_verification_code(- task,- step,- scraped_page,- browser_state,- json_response,+ detailed_agent_step_output.cua_response = new_cua_response+ else:+ using_cached_action_plan = False+ if not task.navigation_goal and not isinstance(task_block, ValidationBlock):+ actions = [await self.create_extract_action(task, step, scraped_page)]+ elif (+ task_block+ and task_block.cache_actions+ and (actions := await retrieve_action_plan(task, step, scraped_page))+ ):+ using_cached_action_plan = True+ else:+ self.async_operation_pool.run_operation(task.task_id, AgentPhase.llm)+ json_response = await app.LLM_API_HANDLER(+ prompt=extract_action_prompt,+ prompt_name="extract-actions",+ step=step,+ screenshots=scraped_page.screenshots,)- detailed_agent_step_output.llm_response = json_response- actions = parse_actions(task, step.step_id, step.order, scraped_page, json_response["actions"])- except NoTOTPVerificationCodeFound:- actions = [- TerminateAction(- organization_id=task.organization_id,- workflow_run_id=task.workflow_run_id,- task_id=task.task_id,- step_id=step.step_id,- step_order=step.order,- action_order=0,- reasoning="No TOTP verification code found. Going to terminate.",- intention="No TOTP verification code found. Going to terminate.",+ try:+ json_response = await self.handle_potential_verification_code(+ task,+ step,+ scraped_page,+ browser_state,+ json_response,)- ]+ detailed_agent_step_output.llm_response = json_response+ actions = parse_actions(task, step.step_id, step.order, scraped_page, json_response["actions"])+ except NoTOTPVerificationCodeFound:+ actions = [+ TerminateAction(+ organization_id=task.organization_id,+ workflow_run_id=task.workflow_run_id,+ task_id=task.task_id,+ step_id=step.step_id,+ step_order=step.order,+ action_order=0,+ reasoning="No TOTP verification code found. Going to terminate.",+ intention="No TOTP verification code found. Going to terminate.",+ )+ ]detailed_agent_step_output.actions = actionsif len(actions) == 0:@@ -1187,6 +1215,77 @@ class ForgeAgent:)return failed_step, detailed_agent_step_output.get_clean_detailed_output()+ async def _generate_cua_actions(+ self,+ task: Task,+ step: Step,+ scraped_page: ScrapedPage,+ previous_response: OpenAIResponse | None = None,+ ) -> tuple[list[Action], OpenAIResponse]:+ if not previous_response:+ # this is the first step+ first_response: OpenAIResponse = await app.OPENAI_CLIENT.responses.create(+ model="computer-use-preview",+ tools=[+ {+ "type": "computer_use_preview",+ "display_width": settings.BROWSER_WIDTH,+ "display_height": settings.BROWSER_HEIGHT,+ "environment": "browser",+ }+ ],+ input=[+ {+ "role": "user",+ "content": task.navigation_goal,+ }+ ],+ reasoning={+ "generate_summary": "concise",+ },+ truncation="auto",+ )+ previous_response = first_response++ computer_calls = [item for item in previous_response.output if item.type == "computer_call"]+ if not computer_calls:+ return [], previous_response++ if not scraped_page.screenshots:+ return [], previous_response++ last_call_id = computer_calls[-1].call_id+ screenshot_base64 = base64.b64encode(scraped_page.screenshots[0]).decode("utf-8")++ current_response = await app.OPENAI_CLIENT.responses.create(+ model="computer-use-preview",+ previous_response_id=previous_response.id,+ tools=[+ {+ "type": "computer_use_preview",+ "display_width": settings.BROWSER_WIDTH,+ "display_height": settings.BROWSER_HEIGHT,+ "environment": "browser",+ }+ ],+ input=[+ {+ "call_id": last_call_id,+ "type": "computer_call_output",+ "output": {+ "type": "input_image",+ "image_url": f"data:image/png;base64,{screenshot_base64}",+ },+ }+ ],+ reasoning={+ "generate_summary": "concise",+ },+ truncation="auto",+ )++ return parse_cua_actions(task, step, current_response), current_response+@staticmethodasync def complete_verify(page: Page, scraped_page: ScrapedPage, task: Task, step: Step) -> CompleteVerifyResult:LOG.info(@@ -1195,7 +1294,12 @@ class ForgeAgent:step_id=step.step_id,workflow_run_id=task.workflow_run_id,)- scraped_page_refreshed = await scraped_page.refresh(draw_boxes=False)+ run_obj = await app.DATABASE.get_run(run_id=task.task_id, organization_id=task.organization_id)+ scroll = True+ if run_obj and run_obj.task_run_type == RunType.openai_cua:+ scroll = False++ scraped_page_refreshed = await scraped_page.refresh(draw_boxes=False, scroll=scroll)verification_prompt = load_prompt_with_elements(scraped_page=scraped_page_refreshed,@@ -1351,6 +1455,7 @@ class ForgeAgent:step: Step,browser_state: BrowserState,scrape_type: ScrapeType,+ engine: RunEngine,) -> ScrapedPage:if scrape_type == ScrapeType.NORMAL:pass@@ -1370,11 +1475,21 @@ class ForgeAgent:)await browser_state.reload_page()+ max_screenshot_number = settings.MAX_NUM_SCREENSHOTS+ draw_boxes = True+ scroll = True+ if engine == RunEngine.openai_cua:+ max_screenshot_number = 1+ draw_boxes = False+ scroll = Falsereturn await scrape_website(browser_state,task.url,app.AGENT_FUNCTION.cleanup_element_tree_factory(task=task, step=step),scrape_exclude=app.scrape_exclude,+ max_screenshot_number=max_screenshot_number,+ draw_boxes=draw_boxes,+ scroll=scroll,)async def build_and_record_step_prompt(@@ -1382,6 +1497,7 @@ class ForgeAgent:task: Task,step: Step,browser_state: BrowserState,+ engine: RunEngine,) -> tuple[ScrapedPage, str]:# start the async tasks while running scrape_websiteself.async_operation_pool.run_operation(task.task_id, AgentPhase.scrape)@@ -1399,6 +1515,7 @@ class ForgeAgent:step=step,browser_state=browser_state,scrape_type=scrape_type,+ engine=engine,)breakexcept (FailedToTakeScreenshot, ScrapingFailed) as e:@@ -1431,14 +1548,16 @@ class ForgeAgent:# TODO: we only use HTML element for now, introduce a way to switch in the futureelement_tree_format = ElementTreeFormat.HTMLelement_tree_in_prompt: str = scraped_page.build_element_tree(element_tree_format)- extract_action_prompt = await self._build_extract_action_prompt(- task,- step,- browser_state,- scraped_page,- verification_code_check=bool(task.totp_verification_url or task.totp_identifier),- expire_verification_code=True,- )+ extract_action_prompt = ""+ if engine != RunEngine.openai_cua:+ extract_action_prompt = await self._build_extract_action_prompt(+ task,+ step,+ browser_state,+ scraped_page,+ verification_code_check=bool(task.totp_verification_url or task.totp_identifier),+ expire_verification_code=True,+ )await app.ARTIFACT_MANAGER.create_artifact(step=step,@@ -2146,9 +2265,14 @@ class ForgeAgent:step_result["actions_result"] = action_result_summarysteps_results.append(step_result)+ run_obj = await app.DATABASE.get_run(run_id=task.task_id, organization_id=task.organization_id)+ scroll = True+ if run_obj and run_obj.task_run_type == RunType.openai_cua:+ scroll = False+screenshots: list[bytes] = []if page is not None:- screenshots = await SkyvernFrame.take_split_screenshots(page=page, url=page.url)+ screenshots = await SkyvernFrame.take_split_screenshots(page=page, url=page.url, scroll=scroll)prompt = prompt_engine.load_prompt("summarize-max-steps-reason",diff --git a/skyvern/forge/app.py b/skyvern/forge/app.pyindex 9ce12a7ca..34fcb1552 100644--- a/skyvern/forge/app.py+++ b/skyvern/forge/app.py@@ -1,6 +1,7 @@from typing import Awaitable, Callablefrom fastapi import FastAPI+from openai import AsyncAzureOpenAI, AsyncOpenAIfrom skyvern.forge.agent import ForgeAgentfrom skyvern.forge.agent_functions import AgentFunction@@ -32,6 +33,15 @@ ARTIFACT_MANAGER = ArtifactManager()BROWSER_MANAGER = BrowserManager()EXPERIMENTATION_PROVIDER: BaseExperimentationProvider = NoOpExperimentationProvider()LLM_API_HANDLER = LLMAPIHandlerFactory.get_llm_api_handler(SettingsManager.get_settings().LLM_KEY)+OPENAI_CLIENT = AsyncOpenAI(api_key=SettingsManager.get_settings().OPENAI_API_KEY)+if SettingsManager.get_settings().ENABLE_AZURE_CUA:+ OPENAI_CLIENT = AsyncAzureOpenAI(+ api_key=SettingsManager.get_settings().AZURE_CUA_API_KEY,+ api_version=SettingsManager.get_settings().AZURE_CUA_API_VERSION,+ azure_endpoint=SettingsManager.get_settings().AZURE_CUA_ENDPOINT,+ azure_deployment=SettingsManager.get_settings().AZURE_CUA_DEPLOYMENT,+ )+SECONDARY_LLM_API_HANDLER = LLMAPIHandlerFactory.get_llm_api_handler(SETTINGS_MANAGER.SECONDARY_LLM_KEY if SETTINGS_MANAGER.SECONDARY_LLM_KEY else SETTINGS_MANAGER.LLM_KEY)diff --git a/skyvern/forge/sdk/executor/async_executor.py b/skyvern/forge/sdk/executor/async_executor.pyindex 9cdc73029..2bc36c156 100644--- a/skyvern/forge/sdk/executor/async_executor.py+++ b/skyvern/forge/sdk/executor/async_executor.py@@ -10,6 +10,7 @@ from skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.schemas.task_v2 import TaskV2Statusfrom skyvern.forge.sdk.schemas.tasks import TaskStatusfrom skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus+from skyvern.schemas.runs import RunEngine, RunTypefrom skyvern.services import task_v2_serviceLOG = structlog.get_logger()@@ -91,6 +92,10 @@ class BackgroundTaskExecutor(AsyncExecutor):status=TaskStatus.running,organization_id=organization_id,)+ run_obj = await app.DATABASE.get_run(run_id=task_id, organization_id=organization_id)+ engine = RunEngine.skyvern_v1+ if run_obj and run_obj.task_run_type == RunType.openai_cua:+ engine = RunEngine.openai_cuacontext: SkyvernContext = skyvern_context.ensure_context()context.task_id = task.task_id@@ -106,6 +111,7 @@ class BackgroundTaskExecutor(AsyncExecutor):api_key,close_browser_on_completion=close_browser_on_completion,browser_session_id=browser_session_id,+ engine=engine,)async def execute_workflow(diff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.pyindex 14149dd05..42a8ccf83 100644--- a/skyvern/forge/sdk/routes/agent_protocol.py+++ b/skyvern/forge/sdk/routes/agent_protocol.py@@ -1465,7 +1465,7 @@ async def run_task(analytics.capture("skyvern-oss-run-task", data={"url": run_request.url})await PermissionCheckerFactory.get_instance().check(current_org, browser_session_id=run_request.browser_session_id)- if run_request.engine == RunEngine.skyvern_v1:+ if run_request.engine in [RunEngine.skyvern_v1, RunEngine.openai_cua]:# create task v1# if there's no url, call task generation first to generate the url, data schema if anyurl = run_request.url@@ -1497,6 +1497,7 @@ async def run_task(task_v1_response = await task_v1_service.run_task(task=task_v1_request,organization=current_org,+ engine=run_request.engine,x_max_steps_override=run_request.max_steps,x_api_key=x_api_key,request=request,@@ -1577,6 +1578,8 @@ async def run_task(publish_workflow=run_request.publish_workflow,),)+ if run_request.engine == RunEngine.openai_cua:+ passraise HTTPException(status_code=400, detail=f"Invalid agent engine: {run_request.engine}")diff --git a/skyvern/schemas/runs.py b/skyvern/schemas/runs.pyindex 9151a676c..4492006a4 100644--- a/skyvern/schemas/runs.py+++ b/skyvern/schemas/runs.py@@ -92,11 +92,13 @@ class RunType(StrEnum):task_v1 = "task_v1"task_v2 = "task_v2"workflow_run = "workflow_run"+ openai_cua = "openai_cua"class RunEngine(StrEnum):skyvern_v1 = "skyvern-1.0"skyvern_v2 = "skyvern-2.0"+ openai_cua = "openai-cua"class RunStatus(StrEnum):diff --git a/skyvern/services/task_v1_service.py b/skyvern/services/task_v1_service.pyindex 41e06bb87..5a319e4cd 100644--- a/skyvern/services/task_v1_service.py+++ b/skyvern/services/task_v1_service.py@@ -13,7 +13,7 @@ from skyvern.forge.sdk.executor.factory import AsyncExecutorFactoryfrom skyvern.forge.sdk.schemas.organizations import Organizationfrom skyvern.forge.sdk.schemas.task_generations import TaskGeneration, TaskGenerationBasefrom skyvern.forge.sdk.schemas.tasks import Task, TaskRequest-from skyvern.schemas.runs import RunType+from skyvern.schemas.runs import RunEngine, RunTypeLOG = structlog.get_logger()@@ -76,6 +76,7 @@ async def generate_task(user_prompt: str, organization: Organization) -> TaskGenasync def run_task(task: TaskRequest,organization: Organization,+ engine: RunEngine = RunEngine.skyvern_v1,x_max_steps_override: int | None = None,x_api_key: str | None = None,request: Request | None = None,@@ -83,8 +84,11 @@ async def run_task() -> Task:created_task = await app.agent.create_task(task, organization.organization_id)url_hash = generate_url_hash(task.url)+ run_type = RunType.task_v1+ if engine == RunEngine.openai_cua:+ run_type = RunType.openai_cuaawait app.DATABASE.create_task_run(- task_run_type=RunType.task_v1,+ task_run_type=run_type,organization_id=organization.organization_id,run_id=created_task.task_id,title=task.title,diff --git a/skyvern/webeye/actions/actions.py b/skyvern/webeye/actions/actions.pyindex 1717dd031..0d3604750 100644--- a/skyvern/webeye/actions/actions.py+++ b/skyvern/webeye/actions/actions.py@@ -28,6 +28,9 @@ class ActionType(StrEnum):RELOAD_PAGE = "reload_page"EXTRACT = "extract"+ SCROLL = "scroll"+ KEYPRESS = "keypress"+ TYPE = "type"def is_web_action(self) -> bool:return self in [@@ -177,6 +180,9 @@ class ClickAction(WebAction):action_type: ActionType = ActionType.CLICKfile_url: str | None = Nonedownload: bool = False+ x: int | None = None+ y: int | None = None+ button: str = "left"def __repr__(self) -> str:return f"ClickAction(element_id={self.element_id}, file_url={self.file_url}, download={self.download})"@@ -240,6 +246,7 @@ class CheckboxAction(WebAction):class WaitAction(Action):action_type: ActionType = ActionType.WAIT+ seconds: int = 20class TerminateAction(DecisiveAction):@@ -258,6 +265,19 @@ class ExtractAction(Action):data_extraction_schema: dict[str, Any] | None = None+class ScrollAction(Action):+ action_type: ActionType = ActionType.SCROLL+ x: int+ y: int+ scroll_x: int+ scroll_y: int+++class KeypressAction(Action):+ action_type: ActionType = ActionType.KEYPRESS+ keys: list[str] = []++class ScrapeResult(BaseModel):"""Scraped response from a webpage, including:diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 7a50db895..b7cdf0492 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -77,6 +77,7 @@ from skyvern.webeye.actions.actions import (CheckboxAction,ClickAction,InputOrSelectContext,+ InputTextAction,ScrapeResult,SelectOption,SelectOptionAction,@@ -392,6 +393,12 @@ def check_for_invalid_web_action(task: Task,step: Step,) -> list[ActionResult]:+ if isinstance(action, ClickAction) and action.x is not None and action.y is not None:+ return []++ if isinstance(action, InputTextAction) and not action.element_id:+ return []+if isinstance(action, WebAction) and action.element_id not in scraped_page.id_to_element_dict:return [ActionFailure(MissingElement(element_id=action.element_id), stop_execution_on_failure=False)]@@ -420,6 +427,36 @@ async def handle_click_action(task: Task,step: Step,) -> list[ActionResult]:+ if action.x is not None and action.y is not None:+ # Find the element at the clicked location using JavaScript evaluation+ element_id = await page.evaluate(+ """data => {+ const element = document.elementFromPoint(data.x, data.y);+ if (!element) return null;++ // Function to get the unique_id attribute of an element+ function getElementUniqueId(element) {+ if (element && element.nodeType === 1) {+ // Check if the element has the unique_id attribute+ if (element.hasAttribute('unique_id')) {+ return element.getAttribute('unique_id');+ }++ // If no unique_id attribute is found, return null+ return null;+ }+ return null;+ }++ return getElementUniqueId(element);+ }""",+ {"x": action.x, "y": action.y},+ )+ LOG.info("Clicked element at location", x=action.x, y=action.y, element_id=element_id, button=action.button)++ await page.mouse.click(x=action.x, y=action.y, button=action.button)+ return [ActionSuccess()]+dom = DomUtil(scraped_page=scraped_page, page=page)skyvern_element = await dom.get_skyvern_element_by_id(action.element_id)await asyncio.sleep(0.3)@@ -591,6 +628,11 @@ async def handle_input_text_action(task: Task,step: Step,) -> list[ActionResult]:+ if not action.element_id:+ # This is a CUA type action+ await page.keyboard.type(action.text)+ return [ActionSuccess()]+dom = DomUtil(scraped_page, page)skyvern_element = await dom.get_skyvern_element_by_id(action.element_id)skyvern_frame = await SkyvernFrame.create_instance(skyvern_element.get_frame())@@ -1348,7 +1390,7 @@ async def handle_wait_action(task: Task,step: Step,) -> list[ActionResult]:- await asyncio.sleep(20)+ await asyncio.sleep(action.seconds)return [ActionFailure(exception=Exception("Wait action is treated as a failure"))]@@ -1422,6 +1464,35 @@ async def handle_extract_action(return [ActionFailure(exception=Exception("No data extraction goal"))]+async def handle_scroll_action(+ action: actions.ScrollAction,+ page: Page,+ scraped_page: ScrapedPage,+ task: Task,+ step: Step,+) -> list[ActionResult]:+ await page.mouse.move(action.x, action.y)+ await page.evaluate(f"window.scrollBy({action.scroll_x}, {action.scroll_y})")+ return [ActionSuccess()]+++async def handle_keypress_action(+ action: actions.KeypressAction,+ page: Page,+ scraped_page: ScrapedPage,+ task: Task,+ step: Step,+) -> list[ActionResult]:+ for key in action.keys:+ if key.lower() == "enter":+ await page.keyboard.press("Enter")+ elif key.lower() == "space":+ await page.keyboard.press(" ")+ else:+ await page.keyboard.press(key)+ return [ActionSuccess()]++ActionHandler.register_action_type(ActionType.SOLVE_CAPTCHA, handle_solve_captcha_action)ActionHandler.register_action_type(ActionType.CLICK, handle_click_action)ActionHandler.register_action_type(ActionType.INPUT_TEXT, handle_input_text_action)@@ -1433,6 +1504,8 @@ ActionHandler.register_action_type(ActionType.WAIT, handle_wait_action)ActionHandler.register_action_type(ActionType.TERMINATE, handle_terminate_action)ActionHandler.register_action_type(ActionType.COMPLETE, handle_complete_action)ActionHandler.register_action_type(ActionType.EXTRACT, handle_extract_action)+ActionHandler.register_action_type(ActionType.SCROLL, handle_scroll_action)+ActionHandler.register_action_type(ActionType.KEYPRESS, handle_keypress_action)async def get_actual_value_of_parameter_if_secret(task: Task, parameter: str) -> Any:diff --git a/skyvern/webeye/actions/models.py b/skyvern/webeye/actions/models.pyindex 33a877e60..6976d6082 100644--- a/skyvern/webeye/actions/models.py+++ b/skyvern/webeye/actions/models.py@@ -2,6 +2,7 @@ from __future__ import annotationsfrom typing import Any+from openai.types.responses.response import Response as OpenAIResponsefrom pydantic import BaseModelfrom skyvern.config import settings@@ -40,6 +41,7 @@ class DetailedAgentStepOutput(BaseModel):action_results: list[ActionResult] | Noneactions_and_results: list[tuple[Action, list[ActionResult]]] | Nonestep_exception: str | None = None+ cua_response: OpenAIResponse | None = Noneclass Config:exclude = ["scraped_page", "extract_action_prompt"]@@ -72,6 +74,7 @@ class DetailedAgentStepOutput(BaseModel):if self.actions_and_results is Noneelse [(action, result) for action, result in self.actions_and_results if result],step_exception=self.step_exception,+ cua_response=self.cua_response,)def to_agent_step_output(self) -> AgentStepOutput:diff --git a/skyvern/webeye/actions/parse_actions.py b/skyvern/webeye/actions/parse_actions.pyindex 37c953e55..b379b35b4 100644--- a/skyvern/webeye/actions/parse_actions.py+++ b/skyvern/webeye/actions/parse_actions.py@@ -1,9 +1,11 @@from typing import Any, Dictimport structlog+from openai.types.responses.response import Response as OpenAIResponsefrom pydantic import ValidationErrorfrom skyvern.exceptions import UnsupportedActionType+from skyvern.forge.sdk.models import Stepfrom skyvern.forge.sdk.schemas.tasks import Taskfrom skyvern.webeye.actions.actions import (Action,@@ -13,7 +15,9 @@ from skyvern.webeye.actions.actions import (CompleteAction,DownloadFileAction,InputTextAction,+ KeypressAction,NullAction,+ ScrollAction,SelectOption,SelectOptionAction,SolveCaptchaAction,@@ -194,3 +198,104 @@ def parse_actions()############################ This part of code might not be needed ############################return actions+++def parse_cua_actions(+ task: Task,+ step: Step,+ response: OpenAIResponse,+) -> list[Action]:+ computer_calls = [item for item in response.output if item.type == "computer_call"]+ reasonings = [item for item in response.output if item.type == "reasoning"]+ actions: list[Action] = []+ for idx, computer_call in enumerate(computer_calls):+ cua_action = computer_call.action+ action_type = cua_action.type+ try:+ reasoning = None+ if idx < len(reasonings):+ try:+ reasoning = reasonings[idx].summary[0].text+ except Exception:+ LOG.exception(+ "Failed to parse reasoning",+ task_id=task.task_id,+ step_id=step.step_id,+ step_order=step.order,+ action_order=idx,+ )++ match action_type:+ case "click":+ button = cua_action.button+ if button != "left" and button != "right":+ button = "left"+ reasoning = reasoning or f"Click at: ({cua_action.x}, {cua_action.y})"+ action = ClickAction(+ element_id="",+ x=cua_action.x,+ y=cua_action.y,+ button=button,+ reasoning=reasoning,+ intention=reasoning,+ response=f"Click at: ({cua_action.x}, {cua_action.y})",+ )+ case "scroll":+ reasoning = reasoning or f"Scroll by: ({cua_action.x}, {cua_action.y})"+ action = ScrollAction(+ element_id="",+ x=cua_action.x,+ y=cua_action.y,+ scroll_x=cua_action.scroll_x,+ scroll_y=cua_action.scroll_y,+ reasoning=reasoning,+ intention=reasoning,+ response=f"Scroll by: ({cua_action.x}, {cua_action.y})",+ )+ case "keypress":+ reasoning_str = f"Press keys: {cua_action.keys}"+ if len(cua_action.keys) == 1:+ reasoning_str = f"Press the '{cua_action.keys[0]}' key"+ reasoning = reasoning or reasoning_str+ action = KeypressAction(+ element_id="",+ keys=cua_action.keys,+ reasoning=reasoning,+ intention=reasoning,+ response=str(cua_action.keys),+ )+ case "type":+ action = InputTextAction(+ element_id="",+ text=cua_action.text,+ reasoning=reasoning,+ intention=reasoning,+ response=cua_action.text,+ )+ case "wait":+ action = WaitAction(+ seconds=5,+ reasoning=reasoning,+ intention=reasoning,+ )+ case _:+ raise ValueError(f"Unsupported action type: {action_type}")+ action.organization_id = task.organization_id+ action.workflow_run_id = task.workflow_run_id+ action.task_id = task.task_id+ action.step_id = step.step_id+ action.step_order = step.order+ action.action_order = idx+ actions.append(action)+ except Exception:+ LOG.exception(+ "Failed to parse action",+ task_id=task.task_id,+ step_id=step.step_id,+ step_order=step.order,+ action_order=idx,+ )+ break+ if not actions:+ return [CompleteAction(reasoning="No actions generated", verified=True)]+ return actionsdiff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.pyindex 0cee02caa..a1d95d756 100644--- a/skyvern/webeye/scraper/scraper.py+++ b/skyvern/webeye/scraper/scraper.py@@ -326,13 +326,14 @@ class ScrapedPage(BaseModel):element["children"] = new_childrenreturn element- async def refresh(self, draw_boxes: bool = True) -> Self:+ async def refresh(self, draw_boxes: bool = True, scroll: bool = True) -> Self:refreshed_page = await scrape_website(browser_state=self._browser_state,url=self.url,cleanup_element_tree=self._clean_up_func,scrape_exclude=self._scrape_exclude,draw_boxes=draw_boxes,+ scroll=scroll,)self.elements = refreshed_page.elementsself.id_to_css_dict = refreshed_page.id_to_css_dict@@ -366,6 +367,8 @@ async def scrape_website(scrape_exclude: ScrapeExcludeFunc | None = None,take_screenshots: bool = True,draw_boxes: bool = True,+ max_screenshot_number: int = settings.MAX_NUM_SCREENSHOTS,+ scroll: bool = True,) -> ScrapedPage:"""************************************************************************************************@@ -397,6 +400,8 @@ async def scrape_website(scrape_exclude=scrape_exclude,take_screenshots=take_screenshots,draw_boxes=draw_boxes,+ max_screenshot_number=max_screenshot_number,+ scroll=scroll,)except Exception as e:# NOTE: MAX_SCRAPING_RETRIES is set to 0 in both staging and production@@ -420,6 +425,8 @@ async def scrape_website(scrape_exclude=scrape_exclude,take_screenshots=take_screenshots,draw_boxes=draw_boxes,+ max_screenshot_number=max_screenshot_number,+ scroll=scroll,)@@ -469,6 +476,8 @@ async def scrape_web_unsafe(scrape_exclude: ScrapeExcludeFunc | None = None,take_screenshots: bool = True,draw_boxes: bool = True,+ max_screenshot_number: int = settings.MAX_NUM_SCREENSHOTS,+ scroll: bool = True,) -> ScrapedPage:"""Asynchronous function that performs web scraping without any built-in error handling. This function is intended@@ -503,7 +512,6 @@ async def scrape_web_unsafe(json_to_html(element, need_skyvern_attrs=False) for element in element_tree_trimmed)token_count = count_tokens(element_tree_trimmed_html_str)- max_screenshot_number = settings.MAX_NUM_SCREENSHOTSif token_count > DEFAULT_MAX_TOKENS:max_screenshot_number = min(max_screenshot_number, 1)@@ -512,6 +520,7 @@ async def scrape_web_unsafe(url=url,draw_boxes=draw_boxes,max_number=max_screenshot_number,+ scroll=scroll,)id_to_css_dict, id_to_element_dict, id_to_frame_dict, id_to_element_hash, hash_to_element_ids = build_element_dict(elementsdiff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.pyindex 0fc1b7779..87b7f422e 100644--- a/skyvern/webeye/utils/page.py+++ b/skyvern/webeye/utils/page.py@@ -98,8 +98,11 @@ class SkyvernFrame:url: str,draw_boxes: bool = False,max_number: int = settings.MAX_NUM_SCREENSHOTS,+ scroll: bool = True,) -> List[bytes]:skyvern_page = await SkyvernFrame.create_instance(frame=page)+ if not scroll:+ return [await SkyvernFrame.take_screenshot(page=skyvern_page.frame, full_page=False)]# page is the main frame and the index must be 0assert isinstance(skyvern_page.frame, Page)