Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
pass reasoning to error code gen (#3494)
LawyZhengea7c54c2716b
diff --git a/skyvern/forge/prompts/skyvern/surface-user-defined-errors.j2 b/skyvern/forge/prompts/skyvern/surface-user-defined-errors.j2index a2740052c..cf036e1ac 100644--- a/skyvern/forge/prompts/skyvern/surface-user-defined-errors.j2+++ b/skyvern/forge/prompts/skyvern/surface-user-defined-errors.j2@@ -1,4 +1,4 @@-You are here to help the user use the error codes and their descriptions to surface user-defined errors based on the screenshots, user goal, user details, action history and the HTML elements.+You are here to help the user use the error codes and their descriptions to surface user-defined errors based on the screenshots, user goal, user details, action history{{ ", context" if reasoning else "" }} and the HTML elements.Do not return any error that's not defined by the user.Reply in JSON format with the following keys:@@ -26,6 +26,11 @@ User details:```{{ navigation_payload_str }}```+{% if reasoning %}+Context:+```+{{ reasoning }}+```{% endif %}Consider the action history and the screenshot together.Action history from previous steps: (note: even if the action history suggests goal is achieved, check the screenshot and the DOM elements to make sure the goal is achieved)diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex ac4347b9a..c8fd8cbf0 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -1791,7 +1791,9 @@ async def handle_terminate_action(step: Step,) -> list[ActionResult]:if task.error_code_mapping:- action.errors = await extract_user_defined_errors(task=task, step=step, scraped_page=scraped_page)+ action.errors = await extract_user_defined_errors(+ task=task, step=step, scraped_page=scraped_page, reasoning=action.reasoning+ )return [ActionSuccess()]@@ -1833,7 +1835,9 @@ async def handle_complete_action(action.verified = Trueif task.error_code_mapping:- action.errors = await extract_user_defined_errors(task=task, step=step, scraped_page=scraped_page)+ action.errors = await extract_user_defined_errors(+ task=task, step=step, scraped_page=scraped_page, reasoning=action.reasoning+ )if not task.data_extraction_goal and verification_result.thoughts:await app.DATABASE.update_task(@@ -3729,7 +3733,9 @@ async def _get_input_or_select_context(return input_or_select_context-async def extract_user_defined_errors(task: Task, step: Step, scraped_page: ScrapedPage) -> list[UserDefinedError]:+async def extract_user_defined_errors(+ task: Task, step: Step, scraped_page: ScrapedPage, reasoning: str | None = None+) -> list[UserDefinedError]:action_history = await get_action_history(task=task, current_step=step)scraped_page_refreshed = await scraped_page.refresh(draw_boxes=False)prompt = prompt_engine.load_prompt(@@ -3741,6 +3747,7 @@ async def extract_user_defined_errors(task: Task, step: Step, scraped_page: Scraaction_history=json.dumps(action_history),error_code_mapping_str=json.dumps(task.error_code_mapping) if task.error_code_mapping else "{}",local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),+ reasoning=reasoning,)json_response = await app.EXTRACTION_LLM_API_HANDLER(prompt=prompt,