Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SkyvernPage actions: Support selector and xpath at the same time. Migrate generate script to use click(selector=f"xpath={xpath}") (#3729)
Shuchang Zhengea2e73f8ccef
diff --git a/skyvern/core/script_generations/generate_script.py b/skyvern/core/script_generations/generate_script.pyindex b984834b0..7b3575769 100644--- a/skyvern/core/script_generations/generate_script.py+++ b/skyvern/core/script_generations/generate_script.py@@ -27,6 +27,7 @@ from skyvern.schemas.workflows import FileStorageTypefrom skyvern.webeye.actions.action_types import ActionTypeLOG = structlog.get_logger(__name__)+GENERATE_CODE_AI_MODE = "proactive"# --------------------------------------------------------------------- #@@ -227,7 +228,7 @@ def _action_to_stmt(act: dict[str, Any], task: dict[str, Any], assign_to_output:"""Turn one Action dict into:- await page.<method>(xpath=..., intention=..., data=context.parameters)+ await page.<method>(selector=..., intention=..., data=context.parameters)Or if assign_to_output is True for extract actions:@@ -239,8 +240,8 @@ def _action_to_stmt(act: dict[str, Any], task: dict[str, Any], assign_to_output:if method in ACTIONS_WITH_XPATH:args.append(cst.Arg(- keyword=cst.Name("xpath"),- value=_value(act["xpath"]),+ keyword=cst.Name("selector"),+ value=_value(f"xpath={act['xpath']}"),whitespace_after_arg=cst.ParenthesizedWhitespace(indent=True,last_line=cst.SimpleWhitespace(INDENT),@@ -248,7 +249,18 @@ def _action_to_stmt(act: dict[str, Any], task: dict[str, Any], assign_to_output:))- if method in ["type", "fill"]:+ if method == "click":+ args.append(+ cst.Arg(+ keyword=cst.Name("ai"),+ value=_value(GENERATE_CODE_AI_MODE),+ whitespace_after_arg=cst.ParenthesizedWhitespace(+ indent=True,+ last_line=cst.SimpleWhitespace(INDENT),+ ),+ )+ )+ elif method in ["type", "fill"]:# Use context.parameters if field_name is available, otherwise fallback to direct valueif act.get("field_name"):text_value = cst.Subscript(@@ -273,8 +285,8 @@ def _action_to_stmt(act: dict[str, Any], task: dict[str, Any], assign_to_output:)args.append(cst.Arg(- keyword=cst.Name("ai_infer"),- value=cst.Name("True"),+ keyword=cst.Name("ai"),+ value=_value(GENERATE_CODE_AI_MODE),whitespace_after_arg=cst.ParenthesizedWhitespace(indent=True,last_line=cst.SimpleWhitespace(INDENT),@@ -330,8 +342,8 @@ def _action_to_stmt(act: dict[str, Any], task: dict[str, Any], assign_to_output:)args.append(cst.Arg(- keyword=cst.Name("ai_infer"),- value=cst.Name("True"),+ keyword=cst.Name("ai"),+ value=_value(GENERATE_CODE_AI_MODE),whitespace_after_arg=cst.ParenthesizedWhitespace(indent=True,last_line=cst.SimpleWhitespace(INDENT),@@ -361,8 +373,8 @@ def _action_to_stmt(act: dict[str, Any], task: dict[str, Any], assign_to_output:)args.append(cst.Arg(- keyword=cst.Name("ai_infer"),- value=cst.Name("True"),+ keyword=cst.Name("ai"),+ value=_value(GENERATE_CODE_AI_MODE),whitespace_after_arg=cst.ParenthesizedWhitespace(indent=True,last_line=cst.SimpleWhitespace(INDENT),diff --git a/skyvern/core/script_generations/skyvern_page.py b/skyvern/core/script_generations/skyvern_page.pyindex 4728078a3..c0b36801a 100644--- a/skyvern/core/script_generations/skyvern_page.py+++ b/skyvern/core/script_generations/skyvern_page.py@@ -36,6 +36,7 @@ from skyvern.webeye.actions.actions import ()from skyvern.webeye.actions.handler import (ActionHandler,+ handle_click_action,handle_complete_action,handle_input_text_action,handle_select_option_action,@@ -72,8 +73,8 @@ class ActionCall:error: Exception | None = None # populated if failed-async def _get_element_id_by_xpath(xpath: str, page: Page) -> str | None:- locator = page.locator(f"xpath={xpath}")+async def _get_element_id_by_selector(selector: str, page: Page) -> str | None:+ locator = page.locator(selector)element_id = await locator.get_attribute("unique_id")return element_id@@ -353,9 +354,12 @@ class SkyvernPage:# Create action record. TODO: store more action fieldskwargs = kwargs or {}# we're using "value" instead of "text" for input text actions interface- xpath = kwargs.get("xpath")+ xpath = Noneif action_type == ActionType.CLICK:- xpath = call_result or xpath+ if isinstance(call_result, str) and "xpath=" in call_result:+ xpath_split_list = call_result.split("xpath=")+ if len(xpath_split_list) > 1:+ xpath = xpath_split_list[1]text = Noneselect_option = Noneresponse: str | None = kwargs.get("response")@@ -470,59 +474,120 @@ class SkyvernPage:# If screenshot creation fails, don't block executionpass+ async def _ai_click(+ self,+ selector: str,+ intention: str,+ data: str | dict[str, Any] | None = None,+ timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,+ ) -> str:+ try:+ # Build the element tree of the current page for the prompt+ context = skyvern_context.ensure_context()+ payload_str = _get_context_data(data)+ refreshed_page = await self.scraped_page.generate_scraped_page_without_screenshots()+ element_tree = refreshed_page.build_element_tree()+ single_click_prompt = prompt_engine.load_prompt(+ template="single-click-action",+ navigation_goal=intention,+ navigation_payload_str=payload_str,+ current_url=self.page.url,+ elements=element_tree,+ local_datetime=datetime.now(context.tz_info or datetime.now().astimezone().tzinfo).isoformat(),+ # user_context=getattr(context, "prompt", None),+ )+ json_response = await app.SINGLE_CLICK_AGENT_LLM_API_HANDLER(+ prompt=single_click_prompt,+ prompt_name="single-click-action",+ organization_id=context.organization_id,+ )+ actions_json = json_response.get("actions", [])+ if actions_json:+ organization_id = context.organization_id if context else None+ task_id = context.task_id if context else None+ step_id = context.step_id if context else None+ task = await app.DATABASE.get_task(task_id, organization_id) if task_id and organization_id else None+ step = await app.DATABASE.get_step(step_id, organization_id) if step_id and organization_id else None+ if organization_id and task and step:+ actions = parse_actions(+ task, step.step_id, step.order, self.scraped_page, json_response.get("actions", [])+ )+ action = actions[0]+ result = await handle_click_action(action, self.page, self.scraped_page, task, step)+ if result and result[-1].success is False:+ raise Exception(result[-1].exception_message)+ xpath = action.get_xpath()+ selector = f"xpath={xpath}" if xpath else selector+ return selector+ except Exception:+ LOG.exception(+ f"Failed to do ai click. Falling back to original selector={selector}, intention={intention}, data={data}"+ )++ locator = self.page.locator(selector)+ await locator.click(timeout=timeout)+ return selector+######### Public Interfaces #########@action_wrap(ActionType.CLICK)- async def click(self, xpath: str, intention: str | None = None, data: str | dict[str, Any] | None = None) -> str:- """Click an element identified by ``xpath``.+ async def click(+ self,+ selector: str,+ intention: str | None = None,+ ai: str | None = "fallback",+ data: str | dict[str, Any] | None = None,+ timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,+ ) -> str:+ """Click an element identified by ``selector``.When ``intention`` and ``data`` are provided a new click action isgenerated via the ``single-click-action`` prompt. The model returns a- fresh xpath based on the current DOM and the updated data for this run.- The browser then clicks the element using this newly generated xpath.+ fresh "xpath=..." selector based on the current DOM and the updated data for this run.+ The browser then clicks the element using this newly generated xpath selector.If the prompt generation or parsing fails for any reason we fall back to- clicking the originally supplied ``xpath``.+ clicking the originally supplied ``selector``."""- new_xpath = xpath-- if intention:+ if ai == "fallback":+ # try to click the element with the original selector first+ error_to_raise = Nonetry:- # Build the element tree of the current page for the prompt- context = skyvern_context.ensure_context()- payload_str = _get_context_data(data)- refreshed_page = await self.scraped_page.generate_scraped_page_without_screenshots()- element_tree = refreshed_page.build_element_tree()- single_click_prompt = prompt_engine.load_prompt(- template="single-click-action",- navigation_goal=intention,- navigation_payload_str=payload_str,- current_url=self.page.url,- elements=element_tree,- local_datetime=datetime.now(context.tz_info or datetime.now().astimezone().tzinfo).isoformat(),- # user_context=getattr(context, "prompt", None),+ locator = self.page.locator(selector)+ await locator.click(timeout=timeout)+ return selector+ except Exception as e:+ error_to_raise = e++ # if the original selector doesn't work, try to click the element with the ai generated selector+ if intention:+ return await self._ai_click(+ selector=selector,+ intention=intention,+ data=data,+ timeout=timeout,)- json_response = await app.SINGLE_CLICK_AGENT_LLM_API_HANDLER(- prompt=single_click_prompt,- prompt_name="single-click-action",- organization_id=context.organization_id,+ if error_to_raise:+ raise error_to_raise+ else:+ return selector+ elif ai == "proactive":+ if intention:+ return await self._ai_click(+ selector=selector,+ intention=intention,+ data=data,+ timeout=timeout,)- actions = json_response.get("actions", [])- if actions:- new_xpath = actions[0].get("xpath", xpath) or xpath- except Exception:- # If anything goes wrong, fall back to the original xpath- new_xpath = xpath-- locator = self.page.locator(f"xpath={new_xpath}")- await locator.click(timeout=5000)- return new_xpath+ locator = self.page.locator(selector)+ await locator.click(timeout=timeout)+ return selector@action_wrap(ActionType.INPUT_TEXT)async def fill(self,- xpath: str,+ selector: str,value: str,- ai_infer: bool = False,+ ai: str | None = "fallback",intention: str | None = None,data: str | dict[str, Any] | None = None,timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,@@ -530,9 +595,9 @@ class SkyvernPage:totp_url: str | None = None,) -> str:return await self._input_text(- xpath=xpath,+ selector=selector,value=value,- ai_infer=ai_infer,+ ai=ai,intention=intention,data=data,timeout=timeout,@@ -543,9 +608,9 @@ class SkyvernPage:@action_wrap(ActionType.INPUT_TEXT)async def type(self,- xpath: str,+ selector: str,value: str,- ai_infer: bool = False,+ ai: str | None = "fallback",intention: str | None = None,data: str | dict[str, Any] | None = None,timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,@@ -553,9 +618,9 @@ class SkyvernPage:totp_url: str | None = None,) -> str:return await self._input_text(- xpath=xpath,+ selector=selector,value=value,- ai_infer=ai_infer,+ ai=ai,intention=intention,data=data,timeout=timeout,@@ -563,28 +628,16 @@ class SkyvernPage:totp_url=totp_url,)- async def _input_text(+ async def _ai_input_text(self,- xpath: str,+ selector: str,value: str,- ai_infer: bool = False,- intention: str | None = None,+ intention: str,data: str | dict[str, Any] | None = None,- timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,totp_identifier: str | None = None,totp_url: str | None = None,+ timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,) -> str:- """Input text into an element identified by ``xpath``.-- When ``intention`` and ``data`` are provided a new input text action is- generated via the `script-generation-input-text-generatiion` prompt. The model returns a- fresh text based on the current DOM and the updated data for this run.- The browser then inputs the text using this newly generated text.-- If the prompt generation or parsing fails for any reason we fall back to- inputting the originally supplied ``text``.- """- # format the text with the actual value of the parameter if it's a secret when running a workflowcontext = skyvern_context.current()value = value or ""transformed_value = value@@ -595,7 +648,7 @@ class SkyvernPage:workflow_run_id = context.workflow_run_id if context else Nonetask = await app.DATABASE.get_task(task_id, organization_id) if task_id and organization_id else Nonestep = await app.DATABASE.get_step(step_id, organization_id) if step_id and organization_id else None- if ai_infer and intention:+ if intention:try:prompt = context.prompt if context else Nonedata = data or {}@@ -624,8 +677,8 @@ class SkyvernPage:refreshed_page = await self.scraped_page.generate_scraped_page_without_screenshots()self.scraped_page = refreshed_page- # get the element_id by the xpath- element_id = await _get_element_id_by_xpath(xpath, self.page)+ # get the element_id by the selector+ element_id = await _get_element_id_by_selector(selector, self.page)script_generation_input_text_prompt = prompt_engine.load_prompt(template="script-generation-input-text-generatiion",intention=intention,@@ -639,10 +692,10 @@ class SkyvernPage:)value = json_response.get("answer", value)except Exception:- LOG.exception(f"Failed to adapt value for input text action on xpath={xpath}, value={value}")+ LOG.exception(f"Failed to adapt value for input text action on selector={selector}, value={value}")if context and context.workflow_run_id:- transformed_value = await _get_actual_value_of_parameter_if_secret(context.workflow_run_id, value)+ transformed_value = await _get_actual_value_of_parameter_if_secret(context.workflow_run_id, str(value))if element_id and organization_id and task and step:action = InputTextAction(@@ -661,20 +714,78 @@ class SkyvernPage:if result and result[-1].success is False:raise Exception(result[-1].exception_message)else:- locator = self.page.locator(f"xpath={xpath}")+ locator = self.page.locator(selector)await handler_utils.input_sequentially(locator, transformed_value, timeout=timeout)return value- @action_wrap(ActionType.UPLOAD_FILE)- async def upload_file(+ async def _input_text(self,- xpath: str,- files: str,- ai_infer: bool = False,+ selector: str,+ value: str,+ ai: str | None = "fallback",intention: str | None = None,data: str | dict[str, Any] | None = None,+ totp_identifier: str | None = None,+ totp_url: str | None = None,+ timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,+ ) -> str:+ """Input text into an element identified by ``selector``.++ When ``intention`` and ``data`` are provided a new input text action is+ generated via the `script-generation-input-text-generation` prompt. The model returns a+ fresh text based on the current DOM and the updated data for this run.+ The browser then inputs the text using this newly generated text.++ If the prompt generation or parsing fails for any reason we fall back to+ inputting the originally supplied ``text``.+ """+ # format the text with the actual value of the parameter if it's a secret when running a workflow+ if ai == "fallback":+ error_to_raise = None+ try:+ locator = self.page.locator(selector)+ await handler_utils.input_sequentially(locator, value, timeout=timeout)+ except Exception as e:+ error_to_raise = e++ if intention:+ return await self._ai_input_text(+ selector=selector,+ value=value,+ intention=intention,+ data=data,+ totp_identifier=totp_identifier,+ totp_url=totp_url,+ timeout=timeout,+ )+ if error_to_raise:+ raise error_to_raise+ else:+ return value+ elif ai == "proactive" and intention:+ return await self._ai_input_text(+ selector=selector,+ value=value,+ intention=intention,+ data=data,+ totp_identifier=totp_identifier,+ totp_url=totp_url,+ timeout=timeout,+ )+ locator = self.page.locator(selector)+ await handler_utils.input_sequentially(locator, value, timeout=timeout)+ return value++ async def _ai_upload_file(+ self,+ selector: str,+ files: str,+ file_path: str,+ intention: str,+ data: str | dict[str, Any] | None = None,+ timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,) -> str:- if ai_infer and intention:+ if intention:try:context = skyvern_context.current()prompt = context.prompt if context else None@@ -692,19 +803,62 @@ class SkyvernPage:)files = json_response.get("answer", files)except Exception:- LOG.exception(f"Failed to adapt value for input text action on xpath={xpath}, file={files}")+ LOG.exception(f"Failed to adapt value for input text action on selector={selector}, file={files}")+ if not files:+ raise ValueError("file url must be provided")+ locator = self.page.locator(selector)+ await locator.set_input_files(file_path, timeout=timeout)+ return files++ @action_wrap(ActionType.UPLOAD_FILE)+ async def upload_file(+ self,+ selector: str,+ files: str,+ ai: str | None = "fallback",+ intention: str | None = None,+ data: str | dict[str, Any] | None = None,+ timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,+ ) -> str:file_path = await download_file(files)- locator = self.page.locator(f"xpath={xpath}")- await locator.set_input_files(file_path)+ if ai == "fallback":+ error_to_raise = None+ try:+ locator = self.page.locator(selector)+ await locator.set_input_files(file_path)+ except Exception as e:+ error_to_raise = e+ if intention:+ return await self._ai_upload_file(+ selector=selector,+ files=files,+ file_path=file_path,+ intention=intention,+ data=data,+ timeout=timeout,+ )+ if error_to_raise:+ raise error_to_raise+ else:+ return files+ elif ai == "proactive" and intention:+ return await self._ai_upload_file(+ selector=selector,+ files=files,+ file_path=file_path,+ intention=intention,+ data=data,+ timeout=timeout,+ )+ locator = self.page.locator(selector)+ await locator.set_input_files(file_path, timeout=timeout)return files- @action_wrap(ActionType.SELECT_OPTION)- async def select_option(+ async def _ai_select_option(self,- xpath: str,+ selector: str,value: str,- ai_infer: bool = False,- intention: str | None = None,+ intention: str,data: str | dict[str, Any] | None = None,timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,) -> str:@@ -713,7 +867,7 @@ class SkyvernPage:if context and context.task_id and context.step_id and context.organization_id:task = await app.DATABASE.get_task(context.task_id, organization_id=context.organization_id)step = await app.DATABASE.get_step(context.step_id, organization_id=context.organization_id)- if ai_infer and intention and task and step:+ if intention and task and step:try:prompt = context.prompt if context else None# data = _get_context_data(data)@@ -735,7 +889,9 @@ class SkyvernPage:prompt_name="single-select-action",organization_id=context.organization_id if context else None,)- actions = parse_actions(task, step.step_id, step.order, self.scraped_page, json_response["actions"])+ actions = parse_actions(+ task, step.step_id, step.order, self.scraped_page, json_response.get("actions", [])+ )if actions:action = actions[0]if not action.option:@@ -750,15 +906,60 @@ class SkyvernPage:)else:LOG.exception(- f"Failed to parse actions for select option action on xpath={xpath}, value={value}"+ f"Failed to parse actions for select option action on selector={selector}, value={value}")except Exception:- LOG.exception(f"Failed to adapt value for select option action on xpath={xpath}, value={value}")+ LOG.exception(+ f"Failed to adapt value for select option action on selector={selector}, value={value}"+ )else:- locator = self.page.locator(f"xpath={xpath}")+ locator = self.page.locator(selector)await locator.select_option(option_value, timeout=timeout)return option_value+ @action_wrap(ActionType.SELECT_OPTION)+ async def select_option(+ self,+ selector: str,+ value: str | None = None,+ label: str | None = None,+ ai: str | None = "fallback",+ intention: str | None = None,+ data: str | dict[str, Any] | None = None,+ timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,+ ) -> str:+ value = value or ""+ if ai == "fallback":+ error_to_raise = None+ try:+ locator = self.page.locator(selector)+ await locator.select_option(value, timeout=timeout)+ except Exception as e:+ error_to_raise = e+ if intention:+ return await self._ai_select_option(+ selector=selector,+ value=value,+ intention=intention,+ data=data,+ timeout=timeout,+ )+ if error_to_raise:+ raise error_to_raise+ else:+ return value+ elif ai == "proactive" and intention:+ return await self._ai_select_option(+ selector=selector,+ value=value,+ intention=intention,+ data=data,+ timeout=timeout,+ )+ locator = self.page.locator(selector)+ await locator.select_option(value, timeout=timeout)+ return value+@action_wrap(ActionType.WAIT)async def wait(self, seconds: float, intention: str | None = None, data: str | dict[str, Any] | None = None@@ -873,9 +1074,7 @@ class SkyvernPage:return result@action_wrap(ActionType.VERIFICATION_CODE)- async def verification_code(- self, xpath: str, intention: str | None = None, data: str | dict[str, Any] | None = None- ) -> None:+ async def verification_code(self, intention: str | None = None, data: str | dict[str, Any] | None = None) -> None:return@action_wrap(ActionType.SCROLL)diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.pyindex 860e50617..919cc1f8c 100644--- a/skyvern/services/script_service.py+++ b/skyvern/services/script_service.py@@ -37,6 +37,7 @@ from skyvern.forge.sdk.workflow.models.block import (ForLoopBlock,HttpRequestBlock,LoginBlock,+ NavigationBlock,PDFParserBlock,SendEmailBlock,TaskBlock,@@ -1176,7 +1177,7 @@ async def run_task(if cache_key and cached_fn:# Auto-create workflow block run and task if workflow_run_id is availableworkflow_run_block_id, task_id, step_id = await _create_workflow_block_run_and_task(- block_type=BlockType.TASK,+ block_type=BlockType.NAVIGATION,prompt=prompt,url=url,label=cache_key,@@ -1201,7 +1202,7 @@ async def run_task(except Exception as e:LOG.exception("Failed to run task block. Falling back to AI run.")await _fallback_to_ai_run(- block_type=BlockType.TASK,+ block_type=BlockType.NAVIGATION,cache_key=cache_key,prompt=prompt,url=url,@@ -1216,7 +1217,7 @@ async def run_task(context.prompt = Noneelse:block_validation_output = await _validate_and_get_output_parameter(label)- task_block = TaskBlock(+ task_block = NavigationBlock(label=block_validation_output.label,output_parameter=block_validation_output.output_parameter,url=url,