Skip to content

Automate browser based workflows with AI

no need to verify completion when we decide to complete according to CUA response; extract data based on CUA reponse as well (#2156)

Shuchang Zheng18f24745d0ac
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 5e63872a2..59f46cf3d 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -1164,6 +1164,8 @@ class ForgeAgent:
and self.step_has_completed_goal(detailed_agent_step_output)
):
working_page = await browser_state.must_get_working_page()
+ # refresh task in case the extracted information is updated previously
+ task = await app.DATABASE.get_task(task.task_id, task.organization_id)
extract_action = await self.create_extract_action(task, step, scraped_page)
extract_results = await ActionHandler.handle_action(
scraped_page, task, step, working_page, extract_action
diff --git a/skyvern/forge/prompts/skyvern/extract-information.j2 b/skyvern/forge/prompts/skyvern/extract-information.j2
index 89cb905cf..5c849affe 100644
--- a/skyvern/forge/prompts/skyvern/extract-information.j2
+++ b/skyvern/forge/prompts/skyvern/extract-information.j2
@@ -13,6 +13,10 @@ If you are trying to extract the href links which are using the jinja style like
User Data Extraction Goal: {{ data_extraction_goal }}
+{% if previous_extracted_information %}
+Previous contexts or thoughts: {{ previous_extracted_information }}
+{% endif %}
+
{% if error_code_mapping_str %}
Use the error codes and their descriptions to return errors in the output, do not return any error that's not defined by the user. Don't return any outputs if the schema doesn't specify an error related field. Here are the descriptions defined by the user: {{ error_code_mapping_str }}
{% endif %}
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 2223c41e0..83e0fec06 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -2986,6 +2986,7 @@ async def extract_information_for_navigation_goal(
html_need_skyvern_attrs=False,
navigation_goal=task.navigation_goal,
navigation_payload=task.navigation_payload,
+ previous_extracted_information=task.extracted_information,
data_extraction_goal=task.data_extraction_goal,
extracted_information_schema=task.extracted_information_schema,
current_url=scraped_page_refreshed.url,
diff --git a/skyvern/webeye/actions/parse_actions.py b/skyvern/webeye/actions/parse_actions.py
index 3c4235509..ef022ff10 100644
--- a/skyvern/webeye/actions/parse_actions.py
+++ b/skyvern/webeye/actions/parse_actions.py
@@ -339,15 +339,23 @@ async def parse_cua_actions(
)
if skyvern_action_type == "complete":
if not task.data_extraction_goal and useful_information:
+ LOG.info(
+ "Updating task with useful information",
+ task_id=task.task_id,
+ organization_id=task.organization_id,
+ useful_information=useful_information,
+ assistant_message=assistant_message,
+ reasoning=reasoning,
+ )
await app.DATABASE.update_task(
task.task_id,
organization_id=task.organization_id,
- extracted_information=useful_information,
+ extracted_information=assistant_message,
)
action = CompleteAction(
reasoning=reasoning,
intention=reasoning,
- verified=False,
+ verified=True,
data_extraction_goal=task.data_extraction_goal,
)
elif skyvern_action_type == "terminate":