Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
introduce action.has_mini_agent (#4295)
Shuchang Zhengba5f0620d589
diff --git a/skyvern/core/script_generations/script_skyvern_page.py b/skyvern/core/script_generations/script_skyvern_page.pyindex e685e3a96..9172b8669 100644--- a/skyvern/core/script_generations/script_skyvern_page.py+++ b/skyvern/core/script_generations/script_skyvern_page.py@@ -15,6 +15,7 @@ from skyvern.forge import appfrom skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.core import skyvern_context+from skyvern.services.otp_service import poll_otp_valuefrom skyvern.utils.url_validators import prepend_scheme_and_validate_urlfrom skyvern.webeye.actions.action_types import ActionTypefrom skyvern.webeye.actions.actions import (@@ -398,15 +399,36 @@ class ScriptSkyvernPage(SkyvernPage):# If screenshot creation fails, don't block executionpass- async def get_actual_value(self, value: str) -> str:+ async def get_actual_value(+ self,+ value: str,+ totp_identifier: str | None = None,+ totp_url: str | None = None,+ ) -> str:"""Input text into an element identified by ``selector``."""context = skyvern_context.ensure_context()if context and context.workflow_run_id:+ task_id = context.task_id+ workflow_run_id = context.workflow_run_id+ organization_id = context.organization_id+ value = get_actual_value_of_parameter_if_secret(workflow_run_id, value)+# support TOTP secret and internal it to TOTP codeis_totp_value = value == "BW_TOTP" or value == "OP_TOTP" or value == "AZ_TOTP"if is_totp_value:value = generate_totp_value(context.workflow_run_id, value)- value = get_actual_value_of_parameter_if_secret(context.workflow_run_id, value)+ elif (totp_identifier or totp_url) and organization_id:+ totp_value = await poll_otp_value(+ organization_id=organization_id,+ task_id=task_id,+ workflow_run_id=workflow_run_id,+ totp_verification_url=totp_url,+ totp_identifier=totp_identifier,+ )+ if totp_value:+ # use the totp verification code+ value = totp_value.value+return valueasync def goto(self, url: str, **kwargs: Any) -> None:diff --git a/skyvern/core/script_generations/skyvern_page.py b/skyvern/core/script_generations/skyvern_page.pyindex c17d53fd3..06e502eec 100644--- a/skyvern/core/script_generations/skyvern_page.py+++ b/skyvern/core/script_generations/skyvern_page.py@@ -96,7 +96,12 @@ class SkyvernPage(Page):timeout = kwargs.pop("timeout", settings.BROWSER_LOADING_TIMEOUT_MS)await self.page.goto(url, timeout=timeout, **kwargs)- async def get_actual_value(self, value: str) -> str:+ async def get_actual_value(+ self,+ value: str,+ totp_identifier: str | None = None,+ totp_url: str | None = None,+ ) -> str:return value######### Public Interfaces #########@@ -390,7 +395,11 @@ class SkyvernPage(Page):error_to_raise = Noneif selector:try:- value = await self.get_actual_value(value)+ value = await self.get_actual_value(+ value,+ totp_identifier=totp_identifier,+ totp_url=totp_url,+ )locator = self.page.locator(selector)await handler_utils.input_sequentially(locator, value, timeout=timeout)return valuediff --git a/skyvern/schemas/scripts.py b/skyvern/schemas/scripts.pyindex 2481e8a48..f110a5f08 100644--- a/skyvern/schemas/scripts.py+++ b/skyvern/schemas/scripts.py@@ -137,6 +137,7 @@ class ScriptBlock(BaseModel):run_signature: str | None = None # The function call code to execute this blockworkflow_run_id: str | None = Noneworkflow_run_block_id: str | None = None+ input_fields: list[str] | None = Nonecreated_at: datetimemodified_at: datetimedeleted_at: datetime | None = Nonediff --git a/skyvern/webeye/actions/actions.py b/skyvern/webeye/actions/actions.pyindex 743f0df6d..b9e066bc5 100644--- a/skyvern/webeye/actions/actions.py+++ b/skyvern/webeye/actions/actions.py@@ -135,10 +135,19 @@ class Action(BaseModel):# TOTP timing information for multi-field TOTP sequencestotp_timing_info: dict[str, Any] | None = None+ # flag indicating whether the action requires mini-agent mode+ has_mini_agent: bool | None = None+created_at: datetime | None = Nonemodified_at: datetime | None = Nonecreated_by: str | None = None+ def set_has_mini_agent(self) -> None:+ """+ Set the has_mini_agent flag to True if any mini-agent is involved when handling the action.+ """+ self.has_mini_agent = True+@classmethoddef validate(cls: Type[T], value: Any) -> T:if isinstance(value, dict):diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 2ea6c140d..d41720465 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -1124,6 +1124,7 @@ async def handle_input_text_action(element_id=skyvern_element.get_id(),action=action,)+ action.set_has_mini_agent()return await handle_select_option_action(select_action, page, scraped_page, task, step)incremental_element: list[dict] = []@@ -1219,6 +1220,7 @@ async def handle_input_text_action(try_to_quit_dropdown = Truetry:# TODO: we don't select by value for the auto completion detect case+ action.set_has_mini_agent()select_result = await sequentially_select_from_dropdown(action=select_action,@@ -1303,6 +1305,7 @@ async def handle_input_text_action(# check the phone number format when type=tel and the text is not a secret valueif not is_secret_value and await skyvern_element.get_attr("type") == "tel":try:+ action.set_has_mini_agent()text = await check_phone_number_format(value=text,action=action,@@ -1373,6 +1376,7 @@ async def handle_input_text_action(if action.totp_timing_info:timing_info = action.totp_timing_infoif timing_info.get("is_totp_sequence"):+ action.set_has_mini_agent()result = await _handle_multi_field_totp_sequence(timing_info, task)if result is not None:return result # Return ActionFailure if TOTP handling failed@@ -1405,6 +1409,7 @@ async def handle_input_text_action(if tag_name == InteractiveElement.INPUT and await skyvern_element.get_attr("type") == "date":try:+ action.set_has_mini_agent()text = await check_date_format(value=text,action=action,@@ -1426,6 +1431,7 @@ async def handle_input_text_action(if not await skyvern_element.is_raw_input():is_location_input = input_or_select_context.is_location_input if input_or_select_context else Falseif input_or_select_context and (await skyvern_element.is_auto_completion_input() or is_location_input):+ action.set_has_mini_agent()if result := await input_or_auto_complete_input(input_or_select_context=input_or_select_context,scraped_page=scraped_page,@@ -1667,6 +1673,7 @@ async def handle_select_option_action(# Confirm if the select action is on the custom option elementif await skyvern_element.is_custom_option():click_action = ClickAction(element_id=action.element_id)+ action.set_has_mini_agent()return await chain_click(task, scraped_page, page, click_action, skyvern_element)if not await skyvern_element.is_selectable():@@ -1775,6 +1782,7 @@ async def handle_select_option_action(action=action,)check_action = CheckboxAction(element_id=action.element_id, is_checked=True)+ action.set_has_mini_agent()return await handle_checkbox_action(check_action, page, scraped_page, task, step)if await skyvern_element.is_radio():@@ -1783,6 +1791,7 @@ async def handle_select_option_action(action=action,)click_action = ClickAction(element_id=action.element_id)+ action.set_has_mini_agent()return await chain_click(task, scraped_page, page, click_action, skyvern_element)# FIXME: maybe there's a case where <input type="button"> could trigger dropdown menu?@@ -1792,6 +1801,7 @@ async def handle_select_option_action(action=action,)click_action = ClickAction(element_id=action.element_id)+ action.set_has_mini_agent()return await chain_click(task, scraped_page, page, click_action, skyvern_element)LOG.info(@@ -3634,6 +3644,7 @@ async def normal_select(step: Step,builder: ElementTreeBuilder,) -> List[ActionResult]:+ action.set_has_mini_agent()try:current_text = await skyvern_element.get_attr("selected")if current_text and (current_text == action.option.label or current_text == action.option.value):