Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Fix extraction prompt templating (#3335)
Shuchang Zheng9a699e70f87f
diff --git a/skyvern/core/script_generations/generate_script.py b/skyvern/core/script_generations/generate_script.pyindex 8d5fb4826..ab14134cc 100644--- a/skyvern/core/script_generations/generate_script.py+++ b/skyvern/core/script_generations/generate_script.py@@ -294,13 +294,25 @@ def _action_to_stmt(act: dict[str, Any], assign_to_output: bool = False) -> cst.args.append(cst.Arg(keyword=cst.Name("prompt"),- value=_value(act["data_extraction_goal"]),+ value=_render_value(act["data_extraction_goal"]),whitespace_after_arg=cst.ParenthesizedWhitespace(indent=True,last_line=cst.SimpleWhitespace(INDENT),),))+ if act.get("data_extraction_schema"):+ args.append(+ cst.Arg(+ keyword=cst.Name("schema"),+ value=_value(act["data_extraction_schema"]),+ whitespace_after_arg=cst.ParenthesizedWhitespace(+ indent=True,+ last_line=cst.SimpleWhitespace(INDENT),+ ),+ comma=cst.Comma(),+ )+ )args.extend([@@ -565,6 +577,14 @@ def _build_extract_statement(block_title: str, block: dict[str, Any]) -> cst.Simlast_line=cst.SimpleWhitespace(INDENT),),),+ cst.Arg(+ keyword=cst.Name("schema"),+ value=_value(block.get("data_schema", "")),+ whitespace_after_arg=cst.ParenthesizedWhitespace(+ indent=True,+ last_line=cst.SimpleWhitespace(INDENT),+ ),+ ),cst.Arg(keyword=cst.Name("cache_key"),value=_value(block_title),diff --git a/skyvern/core/script_generations/skyvern_page.py b/skyvern/core/script_generations/skyvern_page.pyindex d30ef4422..15a57b1f5 100644--- a/skyvern/core/script_generations/skyvern_page.py+++ b/skyvern/core/script_generations/skyvern_page.py@@ -20,7 +20,7 @@ from skyvern.forge.sdk.core import skyvern_contextfrom skyvern.utils.prompt_engine import load_prompt_with_elementsfrom skyvern.webeye.actions import handler_utilsfrom skyvern.webeye.actions.action_types import ActionType-from skyvern.webeye.actions.actions import Action, ActionStatus, SelectOption+from skyvern.webeye.actions.actions import Action, ActionStatus, ExtractAction, SelectOptionfrom skyvern.webeye.browser_factory import BrowserStatefrom skyvern.webeye.scraper.scraper import ScrapedPage, scrape_website@@ -224,6 +224,25 @@ class SkyvernPage:response=response,created_by="script",)+ if action_type == ActionType.EXTRACT:+ action = ExtractAction(+ element_id="",+ action_type=action_type,+ status=status,+ organization_id=context.organization_id,+ workflow_run_id=context.workflow_run_id,+ task_id=context.task_id,+ step_id=context.step_id,+ step_order=0,+ action_order=0,+ intention=intention,+ reasoning=f"Auto-generated action for {action_type.value}",+ data_extraction_goal=kwargs.get("prompt"),+ data_extraction_schema=kwargs.get("schema"),+ option=select_option,+ response=response,+ created_by="script",+ )created_action = await app.DATABASE.create_action(action)return created_actiondiff --git a/skyvern/core/script_generations/transform_workflow_run.py b/skyvern/core/script_generations/transform_workflow_run.pyindex ef0017a36..b10a67959 100644--- a/skyvern/core/script_generations/transform_workflow_run.py+++ b/skyvern/core/script_generations/transform_workflow_run.py@@ -7,6 +7,7 @@ from skyvern.core.script_generations.constants import SCRIPT_TASK_BLOCKSfrom skyvern.forge import appfrom skyvern.schemas.workflows import BlockTypefrom skyvern.services import workflow_service+from skyvern.webeye.actions.action_types import ActionTypeLOG = structlog.get_logger(__name__)@@ -100,6 +101,19 @@ async def transform_workflow_run_to_code_gen_input(workflow_run_id: str, organizfor action in actions:action_dump = action.model_dump()action_dump["xpath"] = action.get_xpath()+ if (+ "data_extraction_goal" in final_dump+ and final_dump["data_extraction_goal"]+ and action.action_type == ActionType.EXTRACT+ ):+ # use the right data extraction goal for the extract action+ action_dump["data_extraction_goal"] = final_dump["data_extraction_goal"]+ if (+ "extracted_information_schema" in final_dump+ and final_dump["extracted_information_schema"]+ and action.action_type == ActionType.EXTRACT+ ):+ action_dump["data_extraction_schema"] = final_dump["extracted_information_schema"]action_dumps.append(action_dump)actions_by_task[run_block.task_id] = action_dumpselse:diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex d65e5114d..f756785c7 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -3010,6 +3010,7 @@ class ForgeAgent:return ExtractAction(reasoning=data_extraction_summary_resp.get("summary", "Extracting information from the page"),data_extraction_goal=task.data_extraction_goal,+ data_extraction_schema=task.extracted_information_schema,organization_id=task.organization_id,task_id=task.task_id,workflow_run_id=task.workflow_run_id,diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.pyindex 74c83028e..1b3a4610f 100644--- a/skyvern/services/script_service.py+++ b/skyvern/services/script_service.py@@ -251,6 +251,7 @@ async def execute_script(async def _create_workflow_block_run_and_task(block_type: BlockType,prompt: str | None = None,+ schema: dict[str, Any] | list | str | None = None,url: str | None = None,) -> tuple[str | None, str | None, str | None]:"""@@ -287,7 +288,8 @@ async def _create_workflow_block_run_and_task(title=f"Script {block_type.value} task",navigation_goal=prompt,data_extraction_goal=prompt if block_type == BlockType.EXTRACTION else None,- navigation_payload={},+ extracted_information_schema=schema,+ navigation_payload=None,status="running",organization_id=organization_id,workflow_run_id=workflow_run_id,@@ -899,6 +901,10 @@ async def _generate_block_code_from_task(continueaction_dump = task_action.model_dump()action_dump["xpath"] = task_action.get_xpath()+ is_data_extraction_goal = "data_extraction_goal" in block_data and "data_extraction_goal" in action_dump+ if is_data_extraction_goal:+ # use the raw data extraction goal which is potentially a template+ action_dump["data_extraction_goal"] = block_data["data_extraction_goal"]actions_to_cache.append(action_dump)if not actions_to_cache:@@ -1157,6 +1163,7 @@ async def login(async def extract(prompt: str,+ schema: dict[str, Any] | list | str | None = None,url: str | None = None,max_steps: int | None = None,cache_key: str | None = None,@@ -1165,6 +1172,7 @@ async def extract(workflow_run_block_id, task_id, step_id = await _create_workflow_block_run_and_task(block_type=BlockType.EXTRACTION,prompt=prompt,+ schema=schema,url=url,)# set the prompt in the RunContext