Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix observer - get url block back; get mark workflow run completion back (#1726)
Shuchang Zhengffdb823401c1
diff --git a/skyvern/forge/sdk/services/observer_service.py b/skyvern/forge/sdk/services/observer_service.pyindex d3210373b..d79380b97 100644--- a/skyvern/forge/sdk/services/observer_service.py+++ b/skyvern/forge/sdk/services/observer_service.py@@ -34,6 +34,7 @@ from skyvern.forge.sdk.workflow.models.block import (ForLoopBlock,NavigationBlock,TaskBlock,+ UrlBlock,)from skyvern.forge.sdk.workflow.models.parameter import PARAMETER_TYPE, ContextParameterfrom skyvern.forge.sdk.workflow.models.workflow import (@@ -51,6 +52,7 @@ from skyvern.forge.sdk.workflow.models.yaml import (ForLoopBlockYAML,NavigationBlockYAML,TaskBlockYAML,+ UrlBlockYAML,WorkflowCreateYAMLRequest,WorkflowDefinitionYAML,)@@ -118,7 +120,9 @@ async def initialize_observer_task(metadata_prompt = prompt_engine.load_prompt("observer_generate_metadata", user_goal=user_prompt, user_url=user_url)metadata_response = await app.LLM_API_HANDLER(- prompt=metadata_prompt, observer_thought=observer_thought, prompt_name="observer-generate-metadata"+ prompt=metadata_prompt,+ observer_thought=observer_thought,+ prompt_name="observer-generate-metadata",)# validateLOG.info(f"Initialized observer initial response: {metadata_response}")@@ -350,160 +354,176 @@ async def run_observer_task_helper(max_iterations = int_max_iterations_override or DEFAULT_MAX_ITERATIONSfor i in range(max_iterations):LOG.info(f"Observer iteration i={i}", workflow_run_id=workflow_run_id, url=url)- try:- browser_state = await app.BROWSER_MANAGER.get_or_create_for_workflow_run(- workflow_run=workflow_run,+ task_type = ""+ plan = ""+ block: BlockTypeVar | None = None+ task_history_record: dict[str, Any] = {}+ context = skyvern_context.ensure_context()++ if i == 0:+ # The first iteration is always a GOTO_URL task+ task_type = "goto_url"+ plan = f"Go to this website: {url}"+ task_history_record = {"type": task_type, "task": plan}+ block, block_yaml_list, parameter_yaml_list = await _generate_goto_url_task(+ workflow_id=workflow_id,url=url,- browser_session_id=browser_session_id,)- scraped_page = await scrape_website(- browser_state,- url,- app.AGENT_FUNCTION.cleanup_element_tree_factory(),- scrape_exclude=app.scrape_exclude,+ task_history_record = {"type": task_type, "task": plan}+ else:+ try:+ browser_state = await app.BROWSER_MANAGER.get_or_create_for_workflow_run(+ workflow_run=workflow_run,+ url=url,+ browser_session_id=browser_session_id,+ )+ scraped_page = await scrape_website(+ browser_state,+ url,+ app.AGENT_FUNCTION.cleanup_element_tree_factory(),+ scrape_exclude=app.scrape_exclude,+ )+ element_tree_in_prompt: str = scraped_page.build_element_tree(ElementTreeFormat.HTML)+ page = await browser_state.get_working_page()+ except Exception:+ LOG.exception(+ "Failed to get browser state or scrape website in observer iteration", iteration=i, url=url+ )+ continue+ current_url = str(+ await SkyvernFrame.evaluate(frame=page, expression="() => document.location.href") if page else url)- element_tree_in_prompt: str = scraped_page.build_element_tree(ElementTreeFormat.HTML)- page = await browser_state.get_working_page()- except Exception:- LOG.exception("Failed to get browser state or scrape website in observer iteration", iteration=i, url=url)- continue- current_url = str(- await SkyvernFrame.evaluate(frame=page, expression="() => document.location.href") if page else url- )- context = skyvern_context.ensure_context()- observer_prompt = prompt_engine.load_prompt(- "observer",- current_url=current_url,- elements=element_tree_in_prompt,- user_goal=user_prompt,- task_history=task_history,- local_datetime=datetime.now(context.tz_info).isoformat(),- )- observer_thought = await app.DATABASE.create_observer_thought(- observer_cruise_id=observer_cruise_id,- organization_id=organization_id,- workflow_run_id=workflow_run.workflow_run_id,- workflow_id=workflow.workflow_id,- workflow_permanent_id=workflow.workflow_permanent_id,- observer_thought_type=ObserverThoughtType.plan,- observer_thought_scenario=ObserverThoughtScenario.generate_plan,- )- observer_response = await app.LLM_API_HANDLER(- prompt=observer_prompt,- screenshots=scraped_page.screenshots,- observer_thought=observer_thought,- prompt_name="observer-generate-plan",- )- LOG.info(- "Observer response",- observer_response=observer_response,- iteration=i,- current_url=current_url,- workflow_run_id=workflow_run_id,- )- # see if the user goal has achieved or not- user_goal_achieved = observer_response.get("user_goal_achieved", False)- observation = observer_response.get("page_info", "")- thoughts: str = observer_response.get("thoughts", "")- plan: str = observer_response.get("plan", "")- task_type: str = observer_response.get("task_type", "")- # Create and save observer thought- await app.DATABASE.update_observer_thought(- observer_thought_id=observer_thought.observer_thought_id,- organization_id=organization_id,- thought=thoughts,- observation=observation,- answer=plan,- output={"task_type": task_type, "user_goal_achieved": user_goal_achieved},- )-- if user_goal_achieved is True:+ observer_prompt = prompt_engine.load_prompt(+ "observer",+ current_url=current_url,+ elements=element_tree_in_prompt,+ user_goal=user_prompt,+ task_history=task_history,+ local_datetime=datetime.now(context.tz_info).isoformat(),+ )+ observer_thought = await app.DATABASE.create_observer_thought(+ observer_cruise_id=observer_cruise_id,+ organization_id=organization_id,+ workflow_run_id=workflow_run.workflow_run_id,+ workflow_id=workflow.workflow_id,+ workflow_permanent_id=workflow.workflow_permanent_id,+ observer_thought_type=ObserverThoughtType.plan,+ observer_thought_scenario=ObserverThoughtScenario.generate_plan,+ )+ observer_response = await app.LLM_API_HANDLER(+ prompt=observer_prompt,+ screenshots=scraped_page.screenshots,+ observer_thought=observer_thought,+ prompt_name="observer",+ )LOG.info(- "User goal achieved. Workflow run will complete. Observer is stopping",+ "Observer response",+ observer_response=observer_response,iteration=i,+ current_url=current_url,workflow_run_id=workflow_run_id,)- observer_task = await _summarize_observer_task(- observer_task=observer_task,- task_history=task_history,- context=context,- screenshots=scraped_page.screenshots,+ # see if the user goal has achieved or not+ user_goal_achieved = observer_response.get("user_goal_achieved", False)+ observation = observer_response.get("page_info", "")+ thoughts: str = observer_response.get("thoughts", "")+ plan = observer_response.get("plan", "")+ task_type = observer_response.get("task_type", "")+ # Create and save observer thought+ await app.DATABASE.update_observer_thought(+ observer_thought_id=observer_thought.observer_thought_id,+ organization_id=organization_id,+ thought=thoughts,+ observation=observation,+ answer=plan,+ output={"task_type": task_type, "user_goal_achieved": user_goal_achieved},)- break- if not plan:- LOG.warning("No plan found in observer response", observer_response=observer_response)- continue+ if user_goal_achieved is True:+ LOG.info(+ "User goal achieved. Workflow run will complete. Observer is stopping",+ iteration=i,+ workflow_run_id=workflow_run_id,+ )+ observer_task = await _summarize_observer_task(+ observer_task=observer_task,+ task_history=task_history,+ context=context,+ screenshots=scraped_page.screenshots,+ )+ break++ if not plan:+ LOG.warning("No plan found in observer response", observer_response=observer_response)+ continue- # parse observer repsonse and run the next task- if not task_type:- LOG.error("No task type found in observer response", observer_response=observer_response)- await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed(- workflow_run_id=workflow_run_id,- failure_reason="Skyvern failed to generate a task. Please try again later.",- )- break+ # parse observer repsonse and run the next task+ if not task_type:+ LOG.error("No task type found in observer response", observer_response=observer_response)+ await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed(+ workflow_run_id=workflow_run_id,+ failure_reason="Skyvern failed to generate a task. Please try again later.",+ )+ break- block: BlockTypeVar | None = None- task_history_record: dict[str, Any] = {}- if task_type == "extract":- block, block_yaml_list, parameter_yaml_list = await _generate_extraction_task(- observer_cruise=observer_task,- workflow_id=workflow_id,- workflow_permanent_id=workflow.workflow_permanent_id,- workflow_run_id=workflow_run_id,- current_url=current_url,- element_tree_in_prompt=element_tree_in_prompt,- data_extraction_goal=plan,- task_history=task_history,- )- task_history_record = {"type": task_type, "task": plan}- elif task_type == "navigate":- original_url = url if i == 0 else None- navigation_goal = MINI_GOAL_TEMPLATE.format(main_goal=user_prompt, mini_goal=plan)- block, block_yaml_list, parameter_yaml_list = await _generate_navigation_task(- workflow_id=workflow_id,- workflow_permanent_id=workflow.workflow_permanent_id,- workflow_run_id=workflow_run_id,- original_url=original_url,- navigation_goal=navigation_goal,- totp_verification_url=observer_task.totp_verification_url,- totp_identifier=observer_task.totp_identifier,- )- task_history_record = {"type": task_type, "task": plan}- elif task_type == "loop":- try:- block, block_yaml_list, parameter_yaml_list, extraction_obj, inner_task = await _generate_loop_task(+ if task_type == "extract":+ block, block_yaml_list, parameter_yaml_list = await _generate_extraction_task(observer_cruise=observer_task,workflow_id=workflow_id,workflow_permanent_id=workflow.workflow_permanent_id,workflow_run_id=workflow_run_id,- plan=plan,- browser_state=browser_state,- original_url=url,- scraped_page=scraped_page,+ current_url=current_url,+ element_tree_in_prompt=element_tree_in_prompt,+ data_extraction_goal=plan,+ task_history=task_history,)- task_history_record = {- "type": task_type,- "task": plan,- "loop_over_values": extraction_obj.get("loop_values"),- "task_inside_the_loop": inner_task,- }- except Exception:- LOG.exception("Failed to generate loop task")+ task_history_record = {"type": task_type, "task": plan}+ elif task_type == "navigate":+ original_url = url if i == 0 else None+ navigation_goal = MINI_GOAL_TEMPLATE.format(main_goal=user_prompt, mini_goal=plan)+ block, block_yaml_list, parameter_yaml_list = await _generate_navigation_task(+ workflow_id=workflow_id,+ workflow_permanent_id=workflow.workflow_permanent_id,+ workflow_run_id=workflow_run_id,+ original_url=original_url,+ navigation_goal=navigation_goal,+ totp_verification_url=observer_task.totp_verification_url,+ totp_identifier=observer_task.totp_identifier,+ )+ task_history_record = {"type": task_type, "task": plan}+ elif task_type == "loop":+ try:+ block, block_yaml_list, parameter_yaml_list, extraction_obj, inner_task = await _generate_loop_task(+ observer_cruise=observer_task,+ workflow_id=workflow_id,+ workflow_permanent_id=workflow.workflow_permanent_id,+ workflow_run_id=workflow_run_id,+ plan=plan,+ browser_state=browser_state,+ original_url=url,+ scraped_page=scraped_page,+ )+ task_history_record = {+ "type": task_type,+ "task": plan,+ "loop_over_values": extraction_obj.get("loop_values"),+ "task_inside_the_loop": inner_task,+ }+ except Exception:+ LOG.exception("Failed to generate loop task")+ await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed(+ workflow_run_id=workflow_run_id,+ failure_reason="Failed to generate the loop.",+ )+ break+ else:+ LOG.info("Unsupported task type", task_type=task_type)await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed(workflow_run_id=workflow_run_id,- failure_reason="Failed to generate the loop.",+ failure_reason=f"Unsupported task block type gets generated: {task_type}",)break- else:- LOG.info("Unsupported task type", task_type=task_type)- await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed(- workflow_run_id=workflow_run_id,- failure_reason=f"Unsupported task block type gets generated: {task_type}",- )- break# generate the extraction taskblock_result = await block.execute_safe(@@ -565,6 +585,11 @@ async def run_observer_task_helper(if block_result.success is True:completion_screenshots = []try:+ browser_state = await app.BROWSER_MANAGER.get_or_create_for_workflow_run(+ workflow_run=workflow_run,+ url=url,+ browser_session_id=browser_session_id,+ )scraped_page = await scrape_website(browser_state,url,@@ -595,7 +620,7 @@ async def run_observer_task_helper(prompt=observer_completion_prompt,screenshots=completion_screenshots,observer_thought=observer_thought,- prompt_name="observer-check-completion",+ prompt_name="observer_check_completion",)LOG.info("Observer completion check response",@@ -900,7 +925,7 @@ async def _generate_loop_task(task_in_loop_metadata_prompt,screenshots=scraped_page.screenshots,observer_thought=observer_thought_task_in_loop,- prompt_name="observer-generate-task-block",+ prompt_name="observer_generate_task_block",)LOG.info("Task in loop metadata response", task_in_loop_metadata_response=task_in_loop_metadata_response)navigation_goal = task_in_loop_metadata_response.get("navigation_goal")@@ -996,7 +1021,7 @@ async def _generate_extraction_task(generate_extraction_task_response = await app.LLM_API_HANDLER(generate_extraction_task_prompt,observer_cruise=observer_cruise,- prompt_name="observer-generate-extraction-task",+ prompt_name="observer_generate_extraction_task",)LOG.info("Data extraction response", data_extraction_response=generate_extraction_task_response)@@ -1067,6 +1092,34 @@ async def _generate_navigation_task()+async def _generate_goto_url_task(+ workflow_id: str,+ url: str,+) -> tuple[UrlBlock, list[BLOCK_YAML_TYPES], list[PARAMETER_YAML_TYPES]]:+ LOG.info("Generating goto url task", url=url)+ # create OutputParameter for the data_extraction block+ label = f"goto_url_{_generate_random_string()}"++ url_block_yaml = UrlBlockYAML(+ label=label,+ url=url,+ )+ output_parameter = await app.WORKFLOW_SERVICE.create_output_parameter_for_block(+ workflow_id=workflow_id,+ block_yaml=url_block_yaml,+ )+ # create UrlBlock+ return (+ UrlBlock(+ label=label,+ url=url,+ output_parameter=output_parameter,+ ),+ [url_block_yaml],+ [],+ )++def _generate_random_string(length: int = 5) -> str:# Use the current timestamp as the seedrandom.seed(os.urandom(16))@@ -1127,13 +1180,14 @@ async def mark_observer_task_as_completed(output: dict[str, Any] | None = None,) -> ObserverTask:observer_task = await app.DATABASE.update_observer_cruise(- observer_cruise_id=observer_cruise_id,+ observer_cruise_id,organization_id=organization_id,status=ObserverTaskStatus.completed,- workflow_run_id=workflow_run_id,summary=summary,output=output,)+ if workflow_run_id:+ await app.WORKFLOW_SERVICE.mark_workflow_run_as_completed(workflow_run_id)# Track observer cruise duration when completedduration_seconds = (datetime.now(UTC) - observer_task.created_at.replace(tzinfo=UTC)).total_seconds()@@ -1245,7 +1299,7 @@ async def _summarize_observer_task(prompt=observer_summary_prompt,screenshots=screenshots,observer_thought=observer_thought,- prompt_name="observer-generate-summary",+ prompt_name="observer_summary",)LOG.info("Observer summary response", observer_summary_resp=observer_summary_resp)