Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
CUA action parser - generate_cua_fallback_actions helper (#2256)
Shuchang Zheng8f466b31082c
diff --git a/skyvern/webeye/actions/parse_actions.py b/skyvern/webeye/actions/parse_actions.pyindex e508a15a6..d7050c6f2 100644--- a/skyvern/webeye/actions/parse_actions.py+++ b/skyvern/webeye/actions/parse_actions.py@@ -354,100 +354,7 @@ async def parse_cua_actions()reasoning = reasonings[0].summary[0].text if reasonings and reasonings[0].summary else Noneassistant_message = assistant_messages[0].content[0].text if assistant_messages else None- fallback_action_prompt = prompt_engine.load_prompt(- "cua-fallback-action",- navigation_goal=task.navigation_goal,- assistant_message=assistant_message,- assistant_reasoning=reasoning,- )-- action_response = await app.LLM_API_HANDLER(- prompt=fallback_action_prompt,- prompt_name="cua-fallback-action",- )- LOG.info("Fallback action response", action_response=action_response)- skyvern_action_type = action_response.get("action")- useful_information = action_response.get("useful_information")- action = WaitAction(- seconds=5,- reasoning=reasoning,- intention=reasoning,- )- if skyvern_action_type == "complete":- LOG.info(- "Updating task with useful information",- task_id=task.task_id,- organization_id=task.organization_id,- useful_information=useful_information,- assistant_message=assistant_message,- reasoning=reasoning,- )- await app.DATABASE.update_task(- task.task_id,- organization_id=task.organization_id,- extracted_information=assistant_message,- )- action = CompleteAction(- reasoning=reasoning,- intention=reasoning,- verified=True,- data_extraction_goal=task.data_extraction_goal,- )- elif skyvern_action_type == "terminate":- action = TerminateAction(- reasoning=reasoning,- intention=reasoning,- )- elif skyvern_action_type == "solve_captcha":- action = SolveCaptchaAction(- reasoning=reasoning,- intention=reasoning,- )- elif skyvern_action_type == "get_verification_code":- if (task.totp_verification_url or task.totp_identifier) and task.organization_id:- LOG.info(- "Getting verification code for CUA",- task_id=task.task_id,- organization_id=task.organization_id,- workflow_run_id=task.workflow_run_id,- totp_verification_url=task.totp_verification_url,- totp_identifier=task.totp_identifier,- )- try:- verification_code = await poll_verification_code(- task.task_id,- task.organization_id,- workflow_run_id=task.workflow_run_id,- totp_verification_url=task.totp_verification_url,- totp_identifier=task.totp_identifier,- )- reasoning = reasoning or f"Received verification code: {verification_code}"- action = VerificationCodeAction(- verification_code=verification_code,- reasoning=reasoning,- intention=reasoning,- )- except NoTOTPVerificationCodeFound:- reasoning_suffix = "No verification code found"- reasoning = f"{reasoning}. {reasoning_suffix}" if reasoning else reasoning_suffix- action = TerminateAction(- reasoning=reasoning,- intention=reasoning,- )-- else:- action = TerminateAction(- reasoning=reasoning,- intention=reasoning,- )-- action.organization_id = task.organization_id- action.workflow_run_id = task.workflow_run_id- action.task_id = task.task_id- action.step_id = step.step_id- action.step_order = step.order- action.action_order = 0- return [action]+ actions = await generate_cua_fallback_actions(task, step, assistant_message, reasoning)return actions@@ -586,3 +493,104 @@ async def parse_anthropic_actions()idx += 1return actions+++async def generate_cua_fallback_actions(+ task: Task,+ step: Step,+ assistant_message: str | None,+ reasoning: str | None,+) -> list[Action]:+ fallback_action_prompt = prompt_engine.load_prompt(+ "cua-fallback-action",+ navigation_goal=task.navigation_goal,+ assistant_message=assistant_message,+ assistant_reasoning=reasoning,+ )++ action_response = await app.LLM_API_HANDLER(+ prompt=fallback_action_prompt,+ prompt_name="cua-fallback-action",+ )+ LOG.info("Fallback action response", action_response=action_response)+ skyvern_action_type = action_response.get("action")+ useful_information = action_response.get("useful_information")+ action = WaitAction(+ seconds=5,+ reasoning=reasoning,+ intention=reasoning,+ )+ if skyvern_action_type == "complete":+ LOG.info(+ "Updating task with useful information",+ task_id=task.task_id,+ organization_id=task.organization_id,+ useful_information=useful_information,+ assistant_message=assistant_message,+ reasoning=reasoning,+ )+ await app.DATABASE.update_task(+ task.task_id,+ organization_id=task.organization_id,+ extracted_information=assistant_message,+ )+ action = CompleteAction(+ reasoning=reasoning,+ intention=reasoning,+ verified=True,+ data_extraction_goal=task.data_extraction_goal,+ )+ elif skyvern_action_type == "terminate":+ action = TerminateAction(+ reasoning=reasoning,+ intention=reasoning,+ )+ elif skyvern_action_type == "solve_captcha":+ action = SolveCaptchaAction(+ reasoning=reasoning,+ intention=reasoning,+ )+ elif skyvern_action_type == "get_verification_code":+ if (task.totp_verification_url or task.totp_identifier) and task.organization_id:+ LOG.info(+ "Getting verification code for CUA",+ task_id=task.task_id,+ organization_id=task.organization_id,+ workflow_run_id=task.workflow_run_id,+ totp_verification_url=task.totp_verification_url,+ totp_identifier=task.totp_identifier,+ )+ try:+ verification_code = await poll_verification_code(+ task.task_id,+ task.organization_id,+ workflow_run_id=task.workflow_run_id,+ totp_verification_url=task.totp_verification_url,+ totp_identifier=task.totp_identifier,+ )+ reasoning = reasoning or f"Received verification code: {verification_code}"+ action = VerificationCodeAction(+ verification_code=verification_code,+ reasoning=reasoning,+ intention=reasoning,+ )+ except NoTOTPVerificationCodeFound:+ reasoning_suffix = "No verification code found"+ reasoning = f"{reasoning}. {reasoning_suffix}" if reasoning else reasoning_suffix+ action = TerminateAction(+ reasoning=reasoning,+ intention=reasoning,+ )+ else:+ action = TerminateAction(+ reasoning=reasoning,+ intention=reasoning,+ )++ action.organization_id = task.organization_id+ action.workflow_run_id = task.workflow_run_id+ action.task_id = task.task_id+ action.step_id = step.step_id+ action.step_order = step.order+ action.action_order = 0+ return [action]