Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
shu/fallback to skyvern actions when cua returns no action (#2141)
Shuchang Zhengdb7f1abeab4c
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 94f14da1b..5e63872a2 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -1319,7 +1319,7 @@ class ForgeAgent:incremental_cached_tokens=cached_tokens if cached_tokens > 0 else None,)- return parse_cua_actions(task, step, current_response), current_response+ return await parse_cua_actions(task, step, current_response), current_response@staticmethodasync def complete_verify(page: Page, scraped_page: ScrapedPage, task: Task, step: Step) -> CompleteVerifyResult:diff --git a/skyvern/forge/prompts/skyvern/cua-fallback-action.j2 b/skyvern/forge/prompts/skyvern/cua-fallback-action.j2new file mode 100644index 000000000..78b9589e5--- /dev/null+++ b/skyvern/forge/prompts/skyvern/cua-fallback-action.j2@@ -0,0 +1,21 @@+The user is trying to achieve a goal in the browser assisted by an browser AI assistant.++According to the AI assistant's feedback, including reasoning and its message, there's no immediate action the assistant can take in the website.++Help the user decide what to do next based on the assistant's message. Here's the list of available actions:+- solve_captcha: the task is blocked by captcha and the assistant is asking the user to solve the captcha+- complete: the user goal has been achieved+- terminate: the user goal cannot be achieved. Terminate the task. Examples: 1) there's not enough data provided to achieve the goal and the assistant is asking the user to provide more information. For examples: login is required and the user has not provided the login credentials or incorrect credentials are provided; a form needs to be filled and a required field is missing. 2) The site is stuck or not loading after multiple attempts+- get_verification_code: the assistant is asking the user to provide a verification code (2FA, MFA or TOTP code)+- other: the assistant is asking the user to do something else++Return the action to take next in the following JSON format:+{+ "action": str // complete, terminate, solve_captcha, get_verification_code+}++User goal: {{ navigation_goal }}++Assistant reasoning: {{ assistant_reasoning }}++Assistant message: {{ assistant_message }}\ No newline at end of filediff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.pyindex b63f83e38..15bbf5ac1 100644--- a/skyvern/forge/sdk/routes/agent_protocol.py+++ b/skyvern/forge/sdk/routes/agent_protocol.py@@ -1479,6 +1479,8 @@ async def run_task()url = url or task_generation.urlnavigation_goal = task_generation.navigation_goal or run_request.prompt+ if run_request.engine == RunEngine.openai_cua:+ navigation_goal = run_request.promptnavigation_payload = task_generation.navigation_payloaddata_extraction_goal = task_generation.data_extraction_goaldata_extraction_schema = data_extraction_schema or task_generation.extracted_information_schemadiff --git a/skyvern/services/run_service.py b/skyvern/services/run_service.pyindex 7022f4012..247d202af 100644--- a/skyvern/services/run_service.py+++ b/skyvern/services/run_service.py@@ -136,7 +136,7 @@ async def cancel_run(run_id: str, organization_id: str | None = None, api_key: sdetail=f"Run not found {run_id}",)- if run.task_run_type == RunType.task_v1:+ if run.task_run_type in [RunType.task_v1, RunType.openai_cua]:await cancel_task_v1(run_id, organization_id=organization_id, api_key=api_key)elif run.task_run_type == RunType.task_v2:await cancel_task_v2(run_id, organization_id=organization_id)diff --git a/skyvern/webeye/actions/parse_actions.py b/skyvern/webeye/actions/parse_actions.pyindex 1d2f28025..64e4f7f49 100644--- a/skyvern/webeye/actions/parse_actions.py+++ b/skyvern/webeye/actions/parse_actions.py@@ -5,6 +5,8 @@ from openai.types.responses.response import Response as OpenAIResponsefrom pydantic import ValidationErrorfrom skyvern.exceptions import UnsupportedActionType+from skyvern.forge import app+from skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.models import Stepfrom skyvern.forge.sdk.schemas.tasks import Taskfrom skyvern.webeye.actions.actions import (@@ -200,13 +202,14 @@ def parse_actions(return actions-def parse_cua_actions(+async def parse_cua_actions(task: Task,step: Step,response: OpenAIResponse,) -> list[Action]:computer_calls = [item for item in response.output if item.type == "computer_call"]reasonings = [item for item in response.output if item.type == "reasoning"]+ assistant_messages = [item for item in response.output if item.type == "message" and item.role == "assistant"]actions: list[Action] = []for idx, computer_call in enumerate(computer_calls):cua_action = computer_call.action@@ -305,16 +308,48 @@ def parse_cua_actions(workflow_run_id=task.workflow_run_id,response=response.dict(),)- complete_action = CompleteAction(- reasoning="No more actions to take",- verified=True,- data_extraction_goal=task.data_extraction_goal,- organization_id=task.organization_id,- workflow_run_id=task.workflow_run_id,- task_id=task.task_id,- step_id=step.step_id,- step_order=step.order,- action_order=0,+ reasoning = reasonings[0].summary[0].text if reasonings and reasonings[0].summary else None+ assistant_message = assistant_messages[0].content[0].text if assistant_messages else None+ fallback_action_prompt = prompt_engine.load_prompt(+ "cua-fallback-action",+ navigation_goal=task.navigation_goal,+ assistant_message=assistant_message,+ assistant_reasoning=reasoning,+ )++ action_response = await app.LLM_API_HANDLER(+ prompt=fallback_action_prompt,+ prompt_name="cua-fallback-action",)- return [complete_action]+ skyvern_action_type = action_response.get("action")+ action = WaitAction(+ seconds=5,+ reasoning=reasoning,+ intention=reasoning,+ )+ if skyvern_action_type == "complete":+ action = CompleteAction(+ reasoning=reasoning,+ intention=reasoning,+ verified=False,+ data_extraction_goal=task.data_extraction_goal,+ )+ elif skyvern_action_type == "terminate":+ action = TerminateAction(+ reasoning=reasoning,+ intention=reasoning,+ )+ elif skyvern_action_type == "solve_captcha":+ action = SolveCaptchaAction(+ reasoning=reasoning,+ intention=reasoning,+ )+ elif skyvern_action_type == "get_verification_code":+ # Currently we don't support verification code+ # TODO: handle verification code by fetching the code and send it to CUA+ action = TerminateAction(+ reasoning=reasoning,+ intention=reasoning,+ )+ return [action]return actions