Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
add history to surface errorcode prompt (#3428)
LawyZheng031d9083a61d
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex ef3980a53..c78351a8f 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -81,6 +81,7 @@ from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowRun, Wofrom skyvern.schemas.runs import CUA_ENGINES, RunEnginefrom skyvern.schemas.steps import AgentStepOutputfrom skyvern.services import run_service+from skyvern.services.action_service import get_action_historyfrom skyvern.services.task_v1_service import is_cua_taskfrom skyvern.utils.image_resizer import Resolutionfrom skyvern.utils.prompt_engine import MaxStepsReasonResponse, load_prompt_with_elements@@ -2100,47 +2101,7 @@ class ForgeAgent:return final_navigation_payloadasync def _get_action_results(self, task: Task, current_step: Step | None = None) -> str:- """- Get the action results from the last app.SETTINGS.PROMPT_ACTION_HISTORY_WINDOW steps.- If current_step is provided, the current executing step will be included in the action history.- Default is excluding the current executing step from the action history.- """-- # Get action results from the last app.SETTINGS.PROMPT_ACTION_HISTORY_WINDOW steps- steps = await app.DATABASE.get_task_steps(task_id=task.task_id, organization_id=task.organization_id)- # the last step is always the newly created one and it should be excluded from the history window- window_steps = steps[-1 - settings.PROMPT_ACTION_HISTORY_WINDOW : -1]- if current_step:- window_steps.append(current_step)-- actions_and_results: list[tuple[Action, list[ActionResult]]] = []- for window_step in window_steps:- if window_step.output and window_step.output.actions_and_results:- actions_and_results.extend(window_step.output.actions_and_results)-- # exclude successful action from history- action_history = [- {- "action": action.model_dump(- exclude_none=True,- include={"action_type", "element_id", "status", "reasoning", "option", "download"},- ),- "results": [- result.model_dump(- exclude_none=True,- include={- "success",- "exception_type",- "exception_message",- },- )- for result in results- ],- }- for action, results in actions_and_results- if len(results) > 0- ]- return json.dumps(action_history)+ return json.dumps(await get_action_history(task=task, current_step=current_step))async def get_extracted_information_for_task(self, task: Task) -> dict[str, Any] | list | str | None:"""diff --git a/skyvern/forge/prompts/skyvern/surface-user-defined-errors.j2 b/skyvern/forge/prompts/skyvern/surface-user-defined-errors.j2index 1bf8a8ebc..a2740052c 100644--- a/skyvern/forge/prompts/skyvern/surface-user-defined-errors.j2+++ b/skyvern/forge/prompts/skyvern/surface-user-defined-errors.j2@@ -1,4 +1,4 @@-You are here to help the user use the error codes and their descriptions to surface user-defined errors based on the screenshots, user goal, user details and the HTML elements.+You are here to help the user use the error codes and their descriptions to surface user-defined errors based on the screenshots, user goal, user details, action history and the HTML elements.Do not return any error that's not defined by the user.Reply in JSON format with the following keys:@@ -27,6 +27,12 @@ User details:{{ navigation_payload_str }}```+Consider the action history and the screenshot together.+Action history from previous steps: (note: even if the action history suggests goal is achieved, check the screenshot and the DOM elements to make sure the goal is achieved)+```+{{action_history}}+```+Clickable elements from `{{ current_url }}`:```{{ elements }}diff --git a/skyvern/services/action_service.py b/skyvern/services/action_service.pynew file mode 100644index 000000000..6b6c73ebb--- /dev/null+++ b/skyvern/services/action_service.py@@ -0,0 +1,54 @@+from typing import Any++from skyvern.config import settings+from skyvern.forge import app+from skyvern.forge.sdk.models import Step+from skyvern.forge.sdk.schemas.tasks import Task+from skyvern.webeye.actions.actions import Action+from skyvern.webeye.actions.responses import ActionResult+++async def get_action_history(+ task: Task, current_step: Step | None = None, history_window: int = settings.PROMPT_ACTION_HISTORY_WINDOW+) -> list[dict[str, Any]]:+ """+ Get the action results from the last history_window steps.+ If current_step is provided, the current executing step will be included in the action history.+ Default is excluding the current executing step from the action history.+ """++ # Get action results from the last history_window steps+ steps = await app.DATABASE.get_task_steps(task_id=task.task_id, organization_id=task.organization_id)+ # the last step is always the newly created one and it should be excluded from the history window+ window_steps = steps[-1 - history_window : -1]+ if current_step:+ window_steps.append(current_step)++ actions_and_results: list[tuple[Action, list[ActionResult]]] = []+ for window_step in window_steps:+ if window_step.output and window_step.output.actions_and_results:+ actions_and_results.extend(window_step.output.actions_and_results)++ # exclude successful action from history+ action_history = [+ {+ "action": action.model_dump(+ exclude_none=True,+ include={"action_type", "element_id", "status", "reasoning", "option", "download"},+ ),+ "results": [+ result.model_dump(+ exclude_none=True,+ include={+ "success",+ "exception_type",+ "exception_message",+ },+ )+ for result in results+ ],+ }+ for action, results in actions_and_results+ if len(results) > 0+ ]+ return action_historydiff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 3dc131d4c..4cdea6fc4 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -69,6 +69,7 @@ from skyvern.forge.sdk.schemas.tasks import Taskfrom skyvern.forge.sdk.services.bitwarden import BitwardenConstantsfrom skyvern.forge.sdk.services.credentials import AzureVaultConstants, OnePasswordConstantsfrom skyvern.forge.sdk.trace import TraceManager+from skyvern.services.action_service import get_action_historyfrom skyvern.services.task_v1_service import is_cua_taskfrom skyvern.utils.prompt_engine import (CheckDateFormatResponse,@@ -3732,6 +3733,7 @@ async def _get_input_or_select_context(async def extract_user_defined_errors(task: Task, step: Step, scraped_page: ScrapedPage) -> list[UserDefinedError]:+ action_history = await get_action_history(task=task, current_step=step)scraped_page_refreshed = await scraped_page.refresh(draw_boxes=False)prompt = prompt_engine.load_prompt("surface-user-defined-errors",@@ -3739,6 +3741,7 @@ async def extract_user_defined_errors(task: Task, step: Step, scraped_page: Scranavigation_payload_str=json.dumps(task.navigation_payload),elements=scraped_page_refreshed.build_element_tree(fmt=ElementTreeFormat.HTML),current_url=scraped_page_refreshed.url,+ action_history=json.dumps(action_history),error_code_mapping_str=json.dumps(task.error_code_mapping) if task.error_code_mapping else "{}",local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),)