Skip to content

Automate browser based workflows with AI

introduce action.has_mini_agent (#4295)

Shuchang Zhengba5f0620d589
diff --git a/skyvern/core/script_generations/script_skyvern_page.py b/skyvern/core/script_generations/script_skyvern_page.py
index e685e3a96..9172b8669 100644
--- a/skyvern/core/script_generations/script_skyvern_page.py
+++ b/skyvern/core/script_generations/script_skyvern_page.py
@@ -15,6 +15,7 @@ from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.core import skyvern_context
+from skyvern.services.otp_service import poll_otp_value
from skyvern.utils.url_validators import prepend_scheme_and_validate_url
from skyvern.webeye.actions.action_types import ActionType
from skyvern.webeye.actions.actions import (
@@ -398,15 +399,36 @@ class ScriptSkyvernPage(SkyvernPage):
# If screenshot creation fails, don't block execution
pass
- async def get_actual_value(self, value: str) -> str:
+ async def get_actual_value(
+ self,
+ value: str,
+ totp_identifier: str | None = None,
+ totp_url: str | None = None,
+ ) -> str:
"""Input text into an element identified by ``selector``."""
context = skyvern_context.ensure_context()
if context and context.workflow_run_id:
+ task_id = context.task_id
+ workflow_run_id = context.workflow_run_id
+ organization_id = context.organization_id
+ value = get_actual_value_of_parameter_if_secret(workflow_run_id, value)
+
# support TOTP secret and internal it to TOTP code
is_totp_value = value == "BW_TOTP" or value == "OP_TOTP" or value == "AZ_TOTP"
if is_totp_value:
value = generate_totp_value(context.workflow_run_id, value)
- value = get_actual_value_of_parameter_if_secret(context.workflow_run_id, value)
+ elif (totp_identifier or totp_url) and organization_id:
+ totp_value = await poll_otp_value(
+ organization_id=organization_id,
+ task_id=task_id,
+ workflow_run_id=workflow_run_id,
+ totp_verification_url=totp_url,
+ totp_identifier=totp_identifier,
+ )
+ if totp_value:
+ # use the totp verification code
+ value = totp_value.value
+
return value
async def goto(self, url: str, **kwargs: Any) -> None:
diff --git a/skyvern/core/script_generations/skyvern_page.py b/skyvern/core/script_generations/skyvern_page.py
index c17d53fd3..06e502eec 100644
--- a/skyvern/core/script_generations/skyvern_page.py
+++ b/skyvern/core/script_generations/skyvern_page.py
@@ -96,7 +96,12 @@ class SkyvernPage(Page):
timeout = kwargs.pop("timeout", settings.BROWSER_LOADING_TIMEOUT_MS)
await self.page.goto(url, timeout=timeout, **kwargs)
- async def get_actual_value(self, value: str) -> str:
+ async def get_actual_value(
+ self,
+ value: str,
+ totp_identifier: str | None = None,
+ totp_url: str | None = None,
+ ) -> str:
return value
######### Public Interfaces #########
@@ -390,7 +395,11 @@ class SkyvernPage(Page):
error_to_raise = None
if selector:
try:
- value = await self.get_actual_value(value)
+ value = await self.get_actual_value(
+ value,
+ totp_identifier=totp_identifier,
+ totp_url=totp_url,
+ )
locator = self.page.locator(selector)
await handler_utils.input_sequentially(locator, value, timeout=timeout)
return value
diff --git a/skyvern/schemas/scripts.py b/skyvern/schemas/scripts.py
index 2481e8a48..f110a5f08 100644
--- a/skyvern/schemas/scripts.py
+++ b/skyvern/schemas/scripts.py
@@ -137,6 +137,7 @@ class ScriptBlock(BaseModel):
run_signature: str | None = None # The function call code to execute this block
workflow_run_id: str | None = None
workflow_run_block_id: str | None = None
+ input_fields: list[str] | None = None
created_at: datetime
modified_at: datetime
deleted_at: datetime | None = None
diff --git a/skyvern/webeye/actions/actions.py b/skyvern/webeye/actions/actions.py
index 743f0df6d..b9e066bc5 100644
--- a/skyvern/webeye/actions/actions.py
+++ b/skyvern/webeye/actions/actions.py
@@ -135,10 +135,19 @@ class Action(BaseModel):
# TOTP timing information for multi-field TOTP sequences
totp_timing_info: dict[str, Any] | None = None
+ # flag indicating whether the action requires mini-agent mode
+ has_mini_agent: bool | None = None
+
created_at: datetime | None = None
modified_at: datetime | None = None
created_by: str | None = None
+ def set_has_mini_agent(self) -> None:
+ """
+ Set the has_mini_agent flag to True if any mini-agent is involved when handling the action.
+ """
+ self.has_mini_agent = True
+
@classmethod
def validate(cls: Type[T], value: Any) -> T:
if isinstance(value, dict):
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 2ea6c140d..d41720465 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -1124,6 +1124,7 @@ async def handle_input_text_action(
element_id=skyvern_element.get_id(),
action=action,
)
+ action.set_has_mini_agent()
return await handle_select_option_action(select_action, page, scraped_page, task, step)
incremental_element: list[dict] = []
@@ -1219,6 +1220,7 @@ async def handle_input_text_action(
try_to_quit_dropdown = True
try:
# TODO: we don't select by value for the auto completion detect case
+ action.set_has_mini_agent()
select_result = await sequentially_select_from_dropdown(
action=select_action,
@@ -1303,6 +1305,7 @@ async def handle_input_text_action(
# check the phone number format when type=tel and the text is not a secret value
if not is_secret_value and await skyvern_element.get_attr("type") == "tel":
try:
+ action.set_has_mini_agent()
text = await check_phone_number_format(
value=text,
action=action,
@@ -1373,6 +1376,7 @@ async def handle_input_text_action(
if action.totp_timing_info:
timing_info = action.totp_timing_info
if timing_info.get("is_totp_sequence"):
+ action.set_has_mini_agent()
result = await _handle_multi_field_totp_sequence(timing_info, task)
if result is not None:
return result # Return ActionFailure if TOTP handling failed
@@ -1405,6 +1409,7 @@ async def handle_input_text_action(
if tag_name == InteractiveElement.INPUT and await skyvern_element.get_attr("type") == "date":
try:
+ action.set_has_mini_agent()
text = await check_date_format(
value=text,
action=action,
@@ -1426,6 +1431,7 @@ async def handle_input_text_action(
if not await skyvern_element.is_raw_input():
is_location_input = input_or_select_context.is_location_input if input_or_select_context else False
if input_or_select_context and (await skyvern_element.is_auto_completion_input() or is_location_input):
+ action.set_has_mini_agent()
if result := await input_or_auto_complete_input(
input_or_select_context=input_or_select_context,
scraped_page=scraped_page,
@@ -1667,6 +1673,7 @@ async def handle_select_option_action(
# Confirm if the select action is on the custom option element
if await skyvern_element.is_custom_option():
click_action = ClickAction(element_id=action.element_id)
+ action.set_has_mini_agent()
return await chain_click(task, scraped_page, page, click_action, skyvern_element)
if not await skyvern_element.is_selectable():
@@ -1775,6 +1782,7 @@ async def handle_select_option_action(
action=action,
)
check_action = CheckboxAction(element_id=action.element_id, is_checked=True)
+ action.set_has_mini_agent()
return await handle_checkbox_action(check_action, page, scraped_page, task, step)
if await skyvern_element.is_radio():
@@ -1783,6 +1791,7 @@ async def handle_select_option_action(
action=action,
)
click_action = ClickAction(element_id=action.element_id)
+ action.set_has_mini_agent()
return await chain_click(task, scraped_page, page, click_action, skyvern_element)
# FIXME: maybe there's a case where <input type="button"> could trigger dropdown menu?
@@ -1792,6 +1801,7 @@ async def handle_select_option_action(
action=action,
)
click_action = ClickAction(element_id=action.element_id)
+ action.set_has_mini_agent()
return await chain_click(task, scraped_page, page, click_action, skyvern_element)
LOG.info(
@@ -3634,6 +3644,7 @@ async def normal_select(
step: Step,
builder: ElementTreeBuilder,
) -> List[ActionResult]:
+ action.set_has_mini_agent()
try:
current_text = await skyvern_element.get_attr("selected")
if current_text and (current_text == action.option.label or current_text == action.option.value):