Skip to content

Automate browser based workflows with AI

update action history prompt (#1932)

Shuchang Zheng26f0e614c4ac
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index e41f93b0a..1f048f5de 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -1542,29 +1542,28 @@ class ForgeAgent:
actions_and_results.extend(window_step.output.actions_and_results)
# exclude successful action from history
- return json.dumps(
- [
- {
- "action": action.model_dump(
+ action_history = [
+ {
+ "action": action.model_dump(
+ exclude_none=True,
+ include={"action_type", "element_id", "status", "reasoning", "option", "download"},
+ ),
+ "results": [
+ result.model_dump(
exclude_none=True,
- include={"action_type", "element_id", "status", "reasoning", "option", "download"},
- ),
- "results": [
- result.model_dump(
- exclude_none=True,
- include={
- "success",
- "exception_type",
- "exception_message",
- },
- )
- for result in results
- ],
- }
- for action, results in actions_and_results
- if len(results) > 0
- ]
- )
+ include={
+ "success",
+ "exception_type",
+ "exception_message",
+ },
+ )
+ for result in results
+ ],
+ }
+ for action, results in actions_and_results
+ if len(results) > 0
+ ]
+ return json.dumps(action_history) if action_history else ""
async def get_extracted_information_for_task(self, task: Task) -> dict[str, Any] | list | str | None:
"""
diff --git a/skyvern/forge/prompts/skyvern/check-user-goal.j2 b/skyvern/forge/prompts/skyvern/check-user-goal.j2
index beb825775..a4eef6b4e 100644
--- a/skyvern/forge/prompts/skyvern/check-user-goal.j2
+++ b/skyvern/forge/prompts/skyvern/check-user-goal.j2
@@ -9,11 +9,6 @@ Make sure to ONLY return the JSON object in this format with no additional text
}
```
-Elements on the page:
-```
-{{ elements }}
-```
-
User Goal:
```
{{ navigation_goal }}
@@ -28,3 +23,8 @@ Complete Criterion:
```
{{ complete_criterion }}
```{% endif %}
+
+Elements on the page:
+```
+{{ elements }}
+```
\ No newline at end of file
diff --git a/skyvern/forge/prompts/skyvern/extract-action.j2 b/skyvern/forge/prompts/skyvern/extract-action.j2
index 4fe0d8e25..4ed00a8c7 100644
--- a/skyvern/forge/prompts/skyvern/extract-action.j2
+++ b/skyvern/forge/prompts/skyvern/extract-action.j2
@@ -42,13 +42,12 @@ Reply in JSON format with the following keys:
}
{% if action_history %}
Consider the action history from the last step and the screenshot together, if actions from the last step don't yield positive impact, try other actions or other action combinations.
-{% endif %}
-Clickable elements from `{{ current_url }}`:
+
+Action history from previous steps: (note: even if the action history suggests goal is achieved, check the screenshot and the DOM elements to make sure the goal is achieved)
```
-{{ elements }}
+{{ action_history }}
```
-
-The URL of the page you're on right now is `{{ current_url }}`.
+{% endif %}
{% if complete_criterion %}
Complete criterion:
```
@@ -74,10 +73,14 @@ User details:
```
{{ navigation_payload_str }}
```
-{% if action_history %}
-Action results from previous steps: (note: even if the action history suggests goal is achieved, check the screenshot and the DOM elements to make sure the goal is achieved)
-{{ action_history }}
-{% endif %}
+
+Clickable elements from `{{ current_url }}`:
+```
+{{ elements }}
+```
+
+The URL of the page you're on right now is `{{ current_url }}`.
+
Current datetime, ISO format:
```
{{ local_datetime }}
diff --git a/skyvern/forge/prompts/skyvern/single-click-action.j2 b/skyvern/forge/prompts/skyvern/single-click-action.j2
index c886df822..0dcb02871 100644
--- a/skyvern/forge/prompts/skyvern/single-click-action.j2
+++ b/skyvern/forge/prompts/skyvern/single-click-action.j2
@@ -19,13 +19,6 @@ Reply in JSON format with the following keys:
}]
}
-The URL of the page you're on right now is `{{ current_url }}`.
-
-HTML elements from `{{ current_url }}`:
-```
-{{ elements }}
-```
-
User instruction (user's intention or self questioning to help figure out what to click):
```
{{ navigation_goal }}
@@ -41,6 +34,13 @@ Context of the big goal user wants to achieve:
{{ user_context }}
```{% endif %}
+The URL of the page you're on right now is `{{ current_url }}`.
+
+HTML elements from `{{ current_url }}`:
+```
+{{ elements }}
+```
+
Current datetime, ISO format:
```
{{ local_datetime }}
diff --git a/skyvern/forge/prompts/skyvern/single-input-action.j2 b/skyvern/forge/prompts/skyvern/single-input-action.j2
index bcf6cec7f..dd0576092 100644
--- a/skyvern/forge/prompts/skyvern/single-input-action.j2
+++ b/skyvern/forge/prompts/skyvern/single-input-action.j2
@@ -20,13 +20,6 @@ Reply in JSON format with the following keys:
"need_verification_code": bool, // Whether a verification code is needed to proceed. True only if the code is available to user. If the code is not sent, return false {% endif %}
}
-The URL of the page you're on right now is `{{ current_url }}`.
-
-HTML elements from `{{ current_url }}`:
-```
-{{ elements }}
-```
-
User instruction:
```
{{ navigation_goal }}
@@ -37,6 +30,13 @@ User details:
{{ navigation_payload_str }}
```
+The URL of the page you're on right now is `{{ current_url }}`.
+
+HTML elements from `{{ current_url }}`:
+```
+{{ elements }}
+```
+
Current datetime, ISO format:
```
{{ local_datetime }}
diff --git a/skyvern/forge/prompts/skyvern/single-select-action.j2 b/skyvern/forge/prompts/skyvern/single-select-action.j2
index 89ae123b0..eea267ad9 100644
--- a/skyvern/forge/prompts/skyvern/single-select-action.j2
+++ b/skyvern/forge/prompts/skyvern/single-select-action.j2
@@ -22,13 +22,6 @@ Reply in JSON format with the following keys:
}]
}
-The URL of the page you're on right now is `{{ current_url }}`.
-
-HTML elements from `{{ current_url }}`:
-```
-{{ elements }}
-```
-
User instruction:
```
{{ navigation_goal }}
@@ -39,6 +32,13 @@ User details:
{{ navigation_payload_str }}
```
+The URL of the page you're on right now is `{{ current_url }}`.
+
+HTML elements from `{{ current_url }}`:
+```
+{{ elements }}
+```
+
Current datetime, ISO format:
```
{{ local_datetime }}
diff --git a/skyvern/forge/prompts/skyvern/single-upload-action.j2 b/skyvern/forge/prompts/skyvern/single-upload-action.j2
index af6c4cb5e..edd4ca533 100644
--- a/skyvern/forge/prompts/skyvern/single-upload-action.j2
+++ b/skyvern/forge/prompts/skyvern/single-upload-action.j2
@@ -18,13 +18,6 @@ Reply in JSON format with the following keys:
}]
}
-The URL of the page you're on right now is `{{ current_url }}`.
-
-HTML elements from `{{ current_url }}`:
-```
-{{ elements }}
-```
-
User instruction:
```
{{ navigation_goal }}
@@ -35,6 +28,13 @@ User details:
{{ navigation_payload_str }}
```
+The URL of the page you're on right now is `{{ current_url }}`.
+
+HTML elements from `{{ current_url }}`:
+```
+{{ elements }}
+```
+
Current datetime, ISO format:
```
{{ local_datetime }}