Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
update action history prompt (#1932)
Shuchang Zheng26f0e614c4ac
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex e41f93b0a..1f048f5de 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -1542,29 +1542,28 @@ class ForgeAgent:actions_and_results.extend(window_step.output.actions_and_results)# exclude successful action from history- return json.dumps(- [- {- "action": action.model_dump(+ action_history = [+ {+ "action": action.model_dump(+ exclude_none=True,+ include={"action_type", "element_id", "status", "reasoning", "option", "download"},+ ),+ "results": [+ result.model_dump(exclude_none=True,- include={"action_type", "element_id", "status", "reasoning", "option", "download"},- ),- "results": [- result.model_dump(- exclude_none=True,- include={- "success",- "exception_type",- "exception_message",- },- )- for result in results- ],- }- for action, results in actions_and_results- if len(results) > 0- ]- )+ include={+ "success",+ "exception_type",+ "exception_message",+ },+ )+ for result in results+ ],+ }+ for action, results in actions_and_results+ if len(results) > 0+ ]+ return json.dumps(action_history) if action_history else ""async def get_extracted_information_for_task(self, task: Task) -> dict[str, Any] | list | str | None:"""diff --git a/skyvern/forge/prompts/skyvern/check-user-goal.j2 b/skyvern/forge/prompts/skyvern/check-user-goal.j2index beb825775..a4eef6b4e 100644--- a/skyvern/forge/prompts/skyvern/check-user-goal.j2+++ b/skyvern/forge/prompts/skyvern/check-user-goal.j2@@ -9,11 +9,6 @@ Make sure to ONLY return the JSON object in this format with no additional text}```-Elements on the page:-```-{{ elements }}-```-User Goal:```{{ navigation_goal }}@@ -28,3 +23,8 @@ Complete Criterion:```{{ complete_criterion }}```{% endif %}++Elements on the page:+```+{{ elements }}+```\ No newline at end of filediff --git a/skyvern/forge/prompts/skyvern/extract-action.j2 b/skyvern/forge/prompts/skyvern/extract-action.j2index 4fe0d8e25..4ed00a8c7 100644--- a/skyvern/forge/prompts/skyvern/extract-action.j2+++ b/skyvern/forge/prompts/skyvern/extract-action.j2@@ -42,13 +42,12 @@ Reply in JSON format with the following keys:}{% if action_history %}Consider the action history from the last step and the screenshot together, if actions from the last step don't yield positive impact, try other actions or other action combinations.-{% endif %}-Clickable elements from `{{ current_url }}`:++Action history from previous steps: (note: even if the action history suggests goal is achieved, check the screenshot and the DOM elements to make sure the goal is achieved)```-{{ elements }}+{{ action_history }}```--The URL of the page you're on right now is `{{ current_url }}`.+{% endif %}{% if complete_criterion %}Complete criterion:```@@ -74,10 +73,14 @@ User details:```{{ navigation_payload_str }}```-{% if action_history %}-Action results from previous steps: (note: even if the action history suggests goal is achieved, check the screenshot and the DOM elements to make sure the goal is achieved)-{{ action_history }}-{% endif %}++Clickable elements from `{{ current_url }}`:+```+{{ elements }}+```++The URL of the page you're on right now is `{{ current_url }}`.+Current datetime, ISO format:```{{ local_datetime }}diff --git a/skyvern/forge/prompts/skyvern/single-click-action.j2 b/skyvern/forge/prompts/skyvern/single-click-action.j2index c886df822..0dcb02871 100644--- a/skyvern/forge/prompts/skyvern/single-click-action.j2+++ b/skyvern/forge/prompts/skyvern/single-click-action.j2@@ -19,13 +19,6 @@ Reply in JSON format with the following keys:}]}-The URL of the page you're on right now is `{{ current_url }}`.--HTML elements from `{{ current_url }}`:-```-{{ elements }}-```-User instruction (user's intention or self questioning to help figure out what to click):```{{ navigation_goal }}@@ -41,6 +34,13 @@ Context of the big goal user wants to achieve:{{ user_context }}```{% endif %}+The URL of the page you're on right now is `{{ current_url }}`.++HTML elements from `{{ current_url }}`:+```+{{ elements }}+```+Current datetime, ISO format:```{{ local_datetime }}diff --git a/skyvern/forge/prompts/skyvern/single-input-action.j2 b/skyvern/forge/prompts/skyvern/single-input-action.j2index bcf6cec7f..dd0576092 100644--- a/skyvern/forge/prompts/skyvern/single-input-action.j2+++ b/skyvern/forge/prompts/skyvern/single-input-action.j2@@ -20,13 +20,6 @@ Reply in JSON format with the following keys:"need_verification_code": bool, // Whether a verification code is needed to proceed. True only if the code is available to user. If the code is not sent, return false {% endif %}}-The URL of the page you're on right now is `{{ current_url }}`.--HTML elements from `{{ current_url }}`:-```-{{ elements }}-```-User instruction:```{{ navigation_goal }}@@ -37,6 +30,13 @@ User details:{{ navigation_payload_str }}```+The URL of the page you're on right now is `{{ current_url }}`.++HTML elements from `{{ current_url }}`:+```+{{ elements }}+```+Current datetime, ISO format:```{{ local_datetime }}diff --git a/skyvern/forge/prompts/skyvern/single-select-action.j2 b/skyvern/forge/prompts/skyvern/single-select-action.j2index 89ae123b0..eea267ad9 100644--- a/skyvern/forge/prompts/skyvern/single-select-action.j2+++ b/skyvern/forge/prompts/skyvern/single-select-action.j2@@ -22,13 +22,6 @@ Reply in JSON format with the following keys:}]}-The URL of the page you're on right now is `{{ current_url }}`.--HTML elements from `{{ current_url }}`:-```-{{ elements }}-```-User instruction:```{{ navigation_goal }}@@ -39,6 +32,13 @@ User details:{{ navigation_payload_str }}```+The URL of the page you're on right now is `{{ current_url }}`.++HTML elements from `{{ current_url }}`:+```+{{ elements }}+```+Current datetime, ISO format:```{{ local_datetime }}diff --git a/skyvern/forge/prompts/skyvern/single-upload-action.j2 b/skyvern/forge/prompts/skyvern/single-upload-action.j2index af6c4cb5e..edd4ca533 100644--- a/skyvern/forge/prompts/skyvern/single-upload-action.j2+++ b/skyvern/forge/prompts/skyvern/single-upload-action.j2@@ -18,13 +18,6 @@ Reply in JSON format with the following keys:}]}-The URL of the page you're on right now is `{{ current_url }}`.--HTML elements from `{{ current_url }}`:-```-{{ elements }}-```-User instruction:```{{ navigation_goal }}@@ -35,6 +28,13 @@ User details:{{ navigation_payload_str }}```+The URL of the page you're on right now is `{{ current_url }}`.++HTML elements from `{{ current_url }}`:+```+{{ elements }}+```+Current datetime, ISO format:```{{ local_datetime }}