Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
enable UrlBlock building in workflow service (#1689)
Shuchang Zheng405d13283c1a
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 33deda2ad..04dbbb469 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -342,6 +342,28 @@ class ForgeAgent:detailed_output,) = await self._initialize_execution_state(task, step, workflow_run, browser_session_id)+ if (+ not task.navigation_goal+ and not task.data_extraction_goal+ and not task.complete_criterion+ and not task.terminate_criterion+ ):+ # most likely a GOTO_URL task block+ # mark step as completed and mark task as completed+ step = await self.update_step(+ step, status=StepStatus.completed, is_last=True, output=AgentStepOutput(action_results=[])+ )+ task = await self.update_task(task, status=TaskStatus.completed)+ await self.clean_up_task(+ task=task,+ last_step=step,+ api_key=api_key,+ need_call_webhook=True,+ close_browser_on_completion=close_browser_on_completion,+ browser_session_id=browser_session_id,+ )+ return step, detailed_output, None+if page := await browser_state.get_working_page():await self.register_async_operations(organization, task, page)diff --git a/skyvern/forge/sdk/models.py b/skyvern/forge/sdk/models.pyindex 556bd2225..e8e460025 100644--- a/skyvern/forge/sdk/models.py+++ b/skyvern/forge/sdk/models.py@@ -18,7 +18,7 @@ class StepStatus(StrEnum):def can_update_to(self, new_status: StepStatus) -> bool:allowed_transitions: dict[StepStatus, set[StepStatus]] = {- StepStatus.created: {StepStatus.running, StepStatus.failed, StepStatus.canceled},+ StepStatus.created: {StepStatus.running, StepStatus.failed, StepStatus.canceled, StepStatus.completed},StepStatus.running: {StepStatus.completed, StepStatus.failed, StepStatus.canceled},StepStatus.failed: set(),StepStatus.completed: set(),@@ -80,9 +80,6 @@ class Step(BaseModel):if self.output is not None and output is not None:raise ValueError(f"cant_override_output({self.step_id})")- if is_last and not self.status.is_terminal():- raise ValueError(f"is_last_but_status_not_terminal({self.status},{self.step_id})")-if is_last is False:raise ValueError(f"cant_set_is_last_to_false({self.step_id})")diff --git a/skyvern/forge/sdk/services/observer_service.py b/skyvern/forge/sdk/services/observer_service.pyindex 06a7e2fae..e8e0bd07b 100644--- a/skyvern/forge/sdk/services/observer_service.py+++ b/skyvern/forge/sdk/services/observer_service.py@@ -34,6 +34,7 @@ from skyvern.forge.sdk.workflow.models.block import (ForLoopBlock,NavigationBlock,TaskBlock,+ UrlBlock,)from skyvern.forge.sdk.workflow.models.parameter import PARAMETER_TYPE, ContextParameterfrom skyvern.forge.sdk.workflow.models.workflow import (@@ -51,6 +52,7 @@ from skyvern.forge.sdk.workflow.models.yaml import (ForLoopBlockYAML,NavigationBlockYAML,TaskBlockYAML,+ UrlBlockYAML,WorkflowCreateYAMLRequest,WorkflowDefinitionYAML,)@@ -348,159 +350,175 @@ async def run_observer_task_helper(max_iterations = int_max_iterations_override or DEFAULT_MAX_ITERATIONSfor i in range(max_iterations):LOG.info(f"Observer iteration i={i}", workflow_run_id=workflow_run_id, url=url)- try:- browser_state = await app.BROWSER_MANAGER.get_or_create_for_workflow_run(- workflow_run=workflow_run,+ task_type = ""+ plan = ""+ block: BlockTypeVar | None = None+ task_history_record: dict[str, Any] = {}+ context = skyvern_context.ensure_context()++ if i == 0:+ # The first iteration is always a GOTO_URL task+ task_type = "goto_url"+ plan = f"Go to this website: {url}"+ task_history_record = {"type": task_type, "task": plan}+ block, block_yaml_list, parameter_yaml_list = await _generate_goto_url_task(+ workflow_id=workflow_id,url=url,- browser_session_id=browser_session_id,)- scraped_page = await scrape_website(- browser_state,- url,- app.AGENT_FUNCTION.cleanup_element_tree_factory(),- scrape_exclude=app.scrape_exclude,+ task_history_record = {"type": task_type, "task": plan}+ else:+ try:+ browser_state = await app.BROWSER_MANAGER.get_or_create_for_workflow_run(+ workflow_run=workflow_run,+ url=url,+ browser_session_id=browser_session_id,+ )+ scraped_page = await scrape_website(+ browser_state,+ url,+ app.AGENT_FUNCTION.cleanup_element_tree_factory(),+ scrape_exclude=app.scrape_exclude,+ )+ element_tree_in_prompt: str = scraped_page.build_element_tree(ElementTreeFormat.HTML)+ page = await browser_state.get_working_page()+ except Exception:+ LOG.exception(+ "Failed to get browser state or scrape website in observer iteration", iteration=i, url=url+ )+ continue+ current_url = str(+ await SkyvernFrame.evaluate(frame=page, expression="() => document.location.href") if page else url)- element_tree_in_prompt: str = scraped_page.build_element_tree(ElementTreeFormat.HTML)- page = await browser_state.get_working_page()- except Exception:- LOG.exception("Failed to get browser state or scrape website in observer iteration", iteration=i, url=url)- continue- current_url = str(- await SkyvernFrame.evaluate(frame=page, expression="() => document.location.href") if page else url- )- context = skyvern_context.ensure_context()- observer_prompt = prompt_engine.load_prompt(- "observer",- current_url=current_url,- elements=element_tree_in_prompt,- user_goal=user_prompt,- task_history=task_history,- local_datetime=datetime.now(context.tz_info).isoformat(),- )- observer_thought = await app.DATABASE.create_observer_thought(- observer_cruise_id=observer_cruise_id,- organization_id=organization_id,- workflow_run_id=workflow_run.workflow_run_id,- workflow_id=workflow.workflow_id,- workflow_permanent_id=workflow.workflow_permanent_id,- observer_thought_type=ObserverThoughtType.plan,- observer_thought_scenario=ObserverThoughtScenario.generate_plan,- )- observer_response = await app.LLM_API_HANDLER(- prompt=observer_prompt,- screenshots=scraped_page.screenshots,- observer_thought=observer_thought,- )- LOG.info(- "Observer response",- observer_response=observer_response,- iteration=i,- current_url=current_url,- workflow_run_id=workflow_run_id,- )- # see if the user goal has achieved or not- user_goal_achieved = observer_response.get("user_goal_achieved", False)- observation = observer_response.get("page_info", "")- thoughts: str = observer_response.get("thoughts", "")- plan: str = observer_response.get("plan", "")- task_type: str = observer_response.get("task_type", "")- # Create and save observer thought- await app.DATABASE.update_observer_thought(- observer_thought_id=observer_thought.observer_thought_id,- organization_id=organization_id,- thought=thoughts,- observation=observation,- answer=plan,- output={"task_type": task_type, "user_goal_achieved": user_goal_achieved},- )-- if user_goal_achieved is True:+ observer_prompt = prompt_engine.load_prompt(+ "observer",+ current_url=current_url,+ elements=element_tree_in_prompt,+ user_goal=user_prompt,+ task_history=task_history,+ local_datetime=datetime.now(context.tz_info).isoformat(),+ )+ observer_thought = await app.DATABASE.create_observer_thought(+ observer_cruise_id=observer_cruise_id,+ organization_id=organization_id,+ workflow_run_id=workflow_run.workflow_run_id,+ workflow_id=workflow.workflow_id,+ workflow_permanent_id=workflow.workflow_permanent_id,+ observer_thought_type=ObserverThoughtType.plan,+ observer_thought_scenario=ObserverThoughtScenario.generate_plan,+ )+ observer_response = await app.LLM_API_HANDLER(+ prompt=observer_prompt,+ screenshots=scraped_page.screenshots,+ observer_thought=observer_thought,+ )LOG.info(- "User goal achieved. Workflow run will complete. Observer is stopping",+ "Observer response",+ observer_response=observer_response,iteration=i,+ current_url=current_url,workflow_run_id=workflow_run_id,)- observer_task = await _summarize_observer_task(- observer_task=observer_task,- task_history=task_history,- context=context,- screenshots=scraped_page.screenshots,+ # see if the user goal has achieved or not+ user_goal_achieved = observer_response.get("user_goal_achieved", False)+ observation = observer_response.get("page_info", "")+ thoughts: str = observer_response.get("thoughts", "")+ plan = observer_response.get("plan", "")+ task_type = observer_response.get("task_type", "")+ # Create and save observer thought+ await app.DATABASE.update_observer_thought(+ observer_thought_id=observer_thought.observer_thought_id,+ organization_id=organization_id,+ thought=thoughts,+ observation=observation,+ answer=plan,+ output={"task_type": task_type, "user_goal_achieved": user_goal_achieved},)- break- if not plan:- LOG.warning("No plan found in observer response", observer_response=observer_response)- continue+ if user_goal_achieved is True:+ LOG.info(+ "User goal achieved. Workflow run will complete. Observer is stopping",+ iteration=i,+ workflow_run_id=workflow_run_id,+ )+ observer_task = await _summarize_observer_task(+ observer_task=observer_task,+ task_history=task_history,+ context=context,+ screenshots=scraped_page.screenshots,+ )+ break- # parse observer repsonse and run the next task- if not task_type:- LOG.error("No task type found in observer response", observer_response=observer_response)- await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed(- workflow_run_id=workflow_run_id,- failure_reason="Skyvern failed to generate a task. Please try again later.",- )- break+ if not plan:+ LOG.warning("No plan found in observer response", observer_response=observer_response)+ continue- block: BlockTypeVar | None = None- task_history_record: dict[str, Any] = {}- if task_type == "extract":- block, block_yaml_list, parameter_yaml_list = await _generate_extraction_task(- observer_cruise=observer_task,- workflow_id=workflow_id,- workflow_permanent_id=workflow.workflow_permanent_id,- workflow_run_id=workflow_run_id,- current_url=current_url,- element_tree_in_prompt=element_tree_in_prompt,- data_extraction_goal=plan,- task_history=task_history,- )- task_history_record = {"type": task_type, "task": plan}- elif task_type == "navigate":- original_url = url if i == 0 else None- navigation_goal = MINI_GOAL_TEMPLATE.format(main_goal=user_prompt, mini_goal=plan)- block, block_yaml_list, parameter_yaml_list = await _generate_navigation_task(- workflow_id=workflow_id,- workflow_permanent_id=workflow.workflow_permanent_id,- workflow_run_id=workflow_run_id,- original_url=original_url,- navigation_goal=navigation_goal,- totp_verification_url=observer_task.totp_verification_url,- totp_identifier=observer_task.totp_identifier,- )- task_history_record = {"type": task_type, "task": plan}- elif task_type == "loop":- try:- block, block_yaml_list, parameter_yaml_list, extraction_obj, inner_task = await _generate_loop_task(+ # parse observer repsonse and run the next task+ if not task_type:+ LOG.error("No task type found in observer response", observer_response=observer_response)+ await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed(+ workflow_run_id=workflow_run_id,+ failure_reason="Skyvern failed to generate a task. Please try again later.",+ )+ break++ if task_type == "extract":+ block, block_yaml_list, parameter_yaml_list = await _generate_extraction_task(observer_cruise=observer_task,workflow_id=workflow_id,workflow_permanent_id=workflow.workflow_permanent_id,workflow_run_id=workflow_run_id,- plan=plan,- browser_state=browser_state,- original_url=url,- scraped_page=scraped_page,+ current_url=current_url,+ element_tree_in_prompt=element_tree_in_prompt,+ data_extraction_goal=plan,+ task_history=task_history,)- task_history_record = {- "type": task_type,- "task": plan,- "loop_over_values": extraction_obj.get("loop_values"),- "task_inside_the_loop": inner_task,- }- except Exception:- LOG.exception("Failed to generate loop task")+ task_history_record = {"type": task_type, "task": plan}+ elif task_type == "navigate":+ original_url = url if i == 0 else None+ navigation_goal = MINI_GOAL_TEMPLATE.format(main_goal=user_prompt, mini_goal=plan)+ block, block_yaml_list, parameter_yaml_list = await _generate_navigation_task(+ workflow_id=workflow_id,+ workflow_permanent_id=workflow.workflow_permanent_id,+ workflow_run_id=workflow_run_id,+ original_url=original_url,+ navigation_goal=navigation_goal,+ totp_verification_url=observer_task.totp_verification_url,+ totp_identifier=observer_task.totp_identifier,+ )+ task_history_record = {"type": task_type, "task": plan}+ elif task_type == "loop":+ try:+ block, block_yaml_list, parameter_yaml_list, extraction_obj, inner_task = await _generate_loop_task(+ observer_cruise=observer_task,+ workflow_id=workflow_id,+ workflow_permanent_id=workflow.workflow_permanent_id,+ workflow_run_id=workflow_run_id,+ plan=plan,+ browser_state=browser_state,+ original_url=url,+ scraped_page=scraped_page,+ )+ task_history_record = {+ "type": task_type,+ "task": plan,+ "loop_over_values": extraction_obj.get("loop_values"),+ "task_inside_the_loop": inner_task,+ }+ except Exception:+ LOG.exception("Failed to generate loop task")+ await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed(+ workflow_run_id=workflow_run_id,+ failure_reason="Failed to generate the loop.",+ )+ break+ else:+ LOG.info("Unsupported task type", task_type=task_type)await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed(workflow_run_id=workflow_run_id,- failure_reason="Failed to generate the loop.",+ failure_reason=f"Unsupported task block type gets generated: {task_type}",)break- else:- LOG.info("Unsupported task type", task_type=task_type)- await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed(- workflow_run_id=workflow_run_id,- failure_reason=f"Unsupported task block type gets generated: {task_type}",- )- break# generate the extraction taskblock_result = await block.execute_safe(@@ -562,6 +580,11 @@ async def run_observer_task_helper(if block_result.success is True:completion_screenshots = []try:+ browser_state = await app.BROWSER_MANAGER.get_or_create_for_workflow_run(+ workflow_run=workflow_run,+ url=url,+ browser_session_id=browser_session_id,+ )scraped_page = await scrape_website(browser_state,url,@@ -1059,6 +1082,34 @@ async def _generate_navigation_task()+async def _generate_goto_url_task(+ workflow_id: str,+ url: str,+) -> tuple[UrlBlock, list[BLOCK_YAML_TYPES], list[PARAMETER_YAML_TYPES]]:+ LOG.info("Generating goto url task", url=url)+ # create OutputParameter for the data_extraction block+ label = f"goto_url_{_generate_random_string()}"++ url_block_yaml = UrlBlockYAML(+ label=label,+ url=url,+ )+ output_parameter = await app.WORKFLOW_SERVICE.create_output_parameter_for_block(+ workflow_id=workflow_id,+ block_yaml=url_block_yaml,+ )+ # create UrlBlock+ return (+ UrlBlock(+ label=label,+ url=url,+ output_parameter=output_parameter,+ ),+ [url_block_yaml],+ [],+ )++def _generate_random_string(length: int = 5) -> str:# Use the current timestamp as the seedrandom.seed(os.urandom(16))diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex a1cba6ce4..cd722119e 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -52,6 +52,7 @@ from skyvern.forge.sdk.workflow.models.block import (TaskV2Block,TextPromptBlock,UploadToS3Block,+ UrlBlock,ValidationBlock,WaitBlock,)@@ -1751,6 +1752,12 @@ class WorkflowService:max_iterations=block_yaml.max_iterations,output_parameter=output_parameter,)+ elif block_yaml.block_type == BlockType.GOTO_URL:+ return UrlBlock(+ label=block_yaml.label,+ url=block_yaml.url,+ output_parameter=output_parameter,+ )raise ValueError(f"Invalid block type {block_yaml.block_type}")