Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat: summarize failure on max retries (#2641)
Shuchang Zhengfdf61aa2f998
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 2ea08384e..e1f5a8eb7 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -2445,10 +2445,25 @@ class ForgeAgent:step_retry=step.retry_index,max_retries=settings.MAX_RETRIES_PER_STEP,)+ browser_state = app.BROWSER_MANAGER.get_for_task(task_id=task.task_id, workflow_run_id=task.workflow_run_id)+ page = None+ if browser_state is not None:+ page = await browser_state.get_working_page()++ failure_reason = await self.summary_failure_reason_for_max_retries(+ organization=organization,+ task=task,+ step=step,+ page=page,+ max_retries=max_retries_per_step,+ )+await self.update_task(task,TaskStatus.failed,- failure_reason=f"Max retries per step ({max_retries_per_step}) exceeded",+ failure_reason=(+ f"Max retries per step ({max_retries_per_step}) exceeded. Possible failure reasons: {failure_reason}"+ ),)return Noneelse:@@ -2530,6 +2545,72 @@ class ForgeAgent:return f"Step {last_step_result['order']}: {last_step_result['actions_result']}"return ""+ async def summary_failure_reason_for_max_retries(+ self,+ organization: Organization,+ task: Task,+ step: Step,+ page: Page | None,+ max_retries: int,+ ) -> str:+ html = ""+ screenshots: list[bytes] = []+ steps_results = []+ try:+ steps = await app.DATABASE.get_task_steps(+ task_id=task.task_id, organization_id=organization.organization_id+ )+ for step_cnt, cur_step in enumerate(steps[-max_retries:]):+ if cur_step.output and cur_step.output.actions_and_results:+ action_result_summary: list[str] = []+ step_result: dict[str, Any] = {+ "order": step_cnt,+ }+ for action, action_results in cur_step.output.actions_and_results:+ if len(action_results) == 0:+ continue+ last_result = action_results[-1]+ if last_result.success:+ continue+ reason = last_result.exception_message or ""+ action_result_summary.append(+ f"{action.reasoning}(action_type={action.action_type}, result=failed, reason={reason})"+ )+ step_result["actions_result"] = action_result_summary+ steps_results.append(step_result)++ if page is not None:+ skyvern_frame = await SkyvernFrame.create_instance(frame=page)+ html = await skyvern_frame.get_content()+ screenshots = await SkyvernFrame.take_split_screenshots(page=page, url=page.url)++ prompt = prompt_engine.load_prompt(+ "summarize-max-retries-reason",+ navigation_goal=task.navigation_goal,+ navigation_payload=task.navigation_payload,+ steps=steps_results,+ page_html=html,+ max_retries=max_retries,+ local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),+ )+ json_response = await app.LLM_API_HANDLER(+ prompt=prompt,+ screenshots=screenshots,+ step=step,+ prompt_name="summarize-max-retries-reason",+ )+ return json_response.get("reasoning", "")+ except Exception:+ LOG.warning(+ "Failed to summarize the failure reason for max retries",+ task_id=task.task_id,+ step_id=step.step_id,+ )+ if steps_results:+ last_step_result = steps_results[-1]+ return f"Retry Step {last_step_result['order']}: {last_step_result['actions_result']}"+ return ""+async def handle_completed_step(self,organization: Organization,diff --git a/skyvern/forge/prompts/skyvern/summarize-max-retries-reason.j2 b/skyvern/forge/prompts/skyvern/summarize-max-retries-reason.j2new file mode 100644index 000000000..c58ca5b25--- /dev/null+++ b/skyvern/forge/prompts/skyvern/summarize-max-retries-reason.j2@@ -0,0 +1,29 @@+User attempted the step multiple times but all {{ max_retries }} retries failed. Summarize the main reason why the actions failed based on the provided screenshot, page HTML, user goal and details.++Make sure to ONLY return the JSON object in this format with no additional text before or after it:+```json+{+ "page_info": str, // Think step by step. Describe useful information from the page HTML related to the user goal.+ "reasoning": str, // Think step by step. Summarize why the actions failed based on 'page_info', screenshots, user goal and the failed actions. Keep it short and to the point.+}+```++User Goal:+{{ navigation_goal }}++User Details:+{{ navigation_payload }}++Failed Actions In Each Retry Step:+{% for step in steps %}Retry Step {{ step.order }} -- {{ step.actions_result }}+{% endfor %}++Page HTML:+```html+{{ page_html }}+```++Current datetime, ISO format:+```+{{ local_datetime }}+```