Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
new observer thoughts (#1442)
Shuchang Zheng9e6c2362bfef
diff --git a/alembic/versions/2024_12_27_1610-d13af1e466fa_new_observer_thoughts.py b/alembic/versions/2024_12_27_1610-d13af1e466fa_new_observer_thoughts.pynew file mode 100644index 000000000..f4ed78e85--- /dev/null+++ b/alembic/versions/2024_12_27_1610-d13af1e466fa_new_observer_thoughts.py@@ -0,0 +1,35 @@+"""new observer thoughts++Revision ID: d13af1e466fa+Revises: 835522a23b19+Create Date: 2024-12-27 16:10:36.555540+00:00++"""++from typing import Sequence, Union++import sqlalchemy as sa++from alembic import op++# revision identifiers, used by Alembic.+revision: str = "d13af1e466fa"+down_revision: Union[str, None] = "835522a23b19"+branch_labels: Union[str, Sequence[str], None] = None+depends_on: Union[str, Sequence[str], None] = None+++def upgrade() -> None:+ # ### commands auto generated by Alembic - please adjust! ###+ op.add_column("observer_thoughts", sa.Column("observer_thought_type", sa.String(), nullable=True))+ op.add_column("observer_thoughts", sa.Column("observer_thought_scenario", sa.String(), nullable=True))+ op.add_column("observer_thoughts", sa.Column("output", sa.JSON(), nullable=True))+ # ### end Alembic commands ###+++def downgrade() -> None:+ # ### commands auto generated by Alembic - please adjust! ###+ op.drop_column("observer_thoughts", "output")+ op.drop_column("observer_thoughts", "observer_thought_scenario")+ op.drop_column("observer_thoughts", "observer_thought_type")+ # ### end Alembic commands ###diff --git a/skyvern/forge/prompts/skyvern/observer_loop_task_extraction_goal.j2 b/skyvern/forge/prompts/skyvern/observer_loop_task_extraction_goal.j2index a537eb7f6..5928b5734 100644--- a/skyvern/forge/prompts/skyvern/observer_loop_task_extraction_goal.j2+++ b/skyvern/forge/prompts/skyvern/observer_loop_task_extraction_goal.j2@@ -1,6 +1,6 @@The user is trying to achieve a goal the web. Now they've decided to go through a list of values and take the same tasks with each variant in the list.-Help to user extract this list of values based on what they want to achieve:+Help the user extract a list of values based on what they want to achieve:```{{ plan }}-```+```\ No newline at end of filediff --git a/skyvern/forge/sdk/db/client.py b/skyvern/forge/sdk/db/client.pyindex bb0ee46e8..bffbb268c 100644--- a/skyvern/forge/sdk/db/client.py+++ b/skyvern/forge/sdk/db/client.py@@ -55,7 +55,12 @@ from skyvern.forge.sdk.db.utils import ()from skyvern.forge.sdk.log_artifacts import save_workflow_run_logsfrom skyvern.forge.sdk.models import Step, StepStatus-from skyvern.forge.sdk.schemas.observers import ObserverCruise, ObserverCruiseStatus, ObserverThought+from skyvern.forge.sdk.schemas.observers import (+ ObserverCruise,+ ObserverCruiseStatus,+ ObserverThought,+ ObserverThoughtType,+)from skyvern.forge.sdk.schemas.organizations import Organization, OrganizationAuthTokenfrom skyvern.forge.sdk.schemas.task_generations import TaskGenerationfrom skyvern.forge.sdk.schemas.tasks import OrderBy, ProxyLocation, SortDirection, Task, TaskStatus@@ -1924,17 +1929,19 @@ class AgentDB:async def get_observer_thoughts(self,observer_cruise_id: str,+ observer_thought_types: list[ObserverThoughtType] | None = None,organization_id: str | None = None,) -> list[ObserverThought]:async with self.Session() as session:- observer_thoughts = (- await session.scalars(- select(ObserverThoughtModel)- .filter_by(observer_cruise_id=observer_cruise_id)- .filter_by(organization_id=organization_id)- .order_by(ObserverThoughtModel.created_at)- )- ).all()+ query = (+ select(ObserverThoughtModel)+ .filter_by(observer_cruise_id=observer_cruise_id)+ .filter_by(organization_id=organization_id)+ .order_by(ObserverThoughtModel.created_at)+ )+ if observer_thought_types:+ query = query.filter(ObserverThoughtModel.observer_thought_type.in_(observer_thought_types))+ observer_thoughts = (await session.scalars(query)).all()return [ObserverThought.model_validate(thought) for thought in observer_thoughts]async def create_observer_cruise(@@ -1971,6 +1978,9 @@ class AgentDB:observation: str | None = None,thought: str | None = None,answer: str | None = None,+ observer_thought_scenario: str | None = None,+ observer_thought_type: str = ObserverThoughtType.plan,+ output: dict[str, Any] | None = None,organization_id: str | None = None,) -> ObserverThought:async with self.Session() as session:@@ -1984,6 +1994,9 @@ class AgentDB:observation=observation,thought=thought,answer=answer,+ observer_thought_scenario=observer_thought_scenario,+ observer_thought_type=observer_thought_type,+ output=output,organization_id=organization_id,)session.add(new_observer_thought)@@ -1995,9 +2008,13 @@ class AgentDB:self,observer_thought_id: str,workflow_run_block_id: str | None = None,+ workflow_run_id: str | None = None,+ workflow_id: str | None = None,+ workflow_permanent_id: str | None = None,observation: str | None = None,thought: str | None = None,answer: str | None = None,+ output: dict[str, Any] | None = None,organization_id: str | None = None,) -> ObserverThought:async with self.Session() as session:@@ -2011,12 +2028,20 @@ class AgentDB:if observer_thought:if workflow_run_block_id:observer_thought.workflow_run_block_id = workflow_run_block_id+ if workflow_run_id:+ observer_thought.workflow_run_id = workflow_run_id+ if workflow_id:+ observer_thought.workflow_id = workflow_id+ if workflow_permanent_id:+ observer_thought.workflow_permanent_id = workflow_permanent_idif observation:observer_thought.observation = observationif thought:observer_thought.thought = thoughtif answer:observer_thought.answer = answer+ if output:+ observer_thought.output = outputawait session.commit()await session.refresh(observer_thought)return ObserverThought.model_validate(observer_thought)diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.pyindex 001f869bd..2966243fc 100644--- a/skyvern/forge/sdk/db/models.py+++ b/skyvern/forge/sdk/db/models.py@@ -41,6 +41,7 @@ from skyvern.forge.sdk.db.id import (generate_workflow_run_block_id,generate_workflow_run_id,)+from skyvern.forge.sdk.schemas.observers import ObserverThoughtTypefrom skyvern.forge.sdk.schemas.tasks import ProxyLocation@@ -559,6 +560,10 @@ class ObserverThoughtModel(Base):thought = Column(String, nullable=True)answer = Column(String, nullable=True)+ observer_thought_type = Column(String, nullable=True, default=ObserverThoughtType.plan)+ observer_thought_scenario = Column(String, nullable=True)+ output = Column(JSON, nullable=True)+created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False)modified_at = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow, nullable=False)diff --git a/skyvern/forge/sdk/schemas/observers.py b/skyvern/forge/sdk/schemas/observers.pyindex 8fc2bf7f6..29d583edd 100644--- a/skyvern/forge/sdk/schemas/observers.py+++ b/skyvern/forge/sdk/schemas/observers.py@@ -1,5 +1,6 @@from datetime import datetimefrom enum import StrEnum+from typing import Anyfrom pydantic import BaseModel, ConfigDict, HttpUrl, field_validator@@ -35,6 +36,22 @@ class ObserverCruise(BaseModel):modified_at: datetime+class ObserverThoughtType(StrEnum):+ plan = "plan"+ metadata = "metadata"+ user_goal_check = "user_goal_check"+ internal_plan = "internal_plan"+++class ObserverThoughtScenario(StrEnum):+ generate_plan = "generate_plan"+ user_goal_check = "user_goal_check"+ generate_metadata = "generate_metadata"+ extract_loop_values = "extract_loop_values"+ generate_task_in_loop = "generate_task_in_loop"+ generate_task = "generate_general_task"++class ObserverThought(BaseModel):model_config = ConfigDict(from_attributes=True)@@ -49,6 +66,9 @@ class ObserverThought(BaseModel):observation: str | None = Nonethought: str | None = Noneanswer: str | None = None+ observer_thought_type: ObserverThoughtType | None = ObserverThoughtType.plan+ observer_thought_scenario: ObserverThoughtScenario | None = None+ output: dict[str, Any] | None = Nonecreated_at: datetimemodified_at: datetimediff --git a/skyvern/forge/sdk/services/observer_service.py b/skyvern/forge/sdk/services/observer_service.pyindex 92de49b0e..b4f3c18e0 100644--- a/skyvern/forge/sdk/services/observer_service.py+++ b/skyvern/forge/sdk/services/observer_service.py@@ -10,9 +10,16 @@ from pydantic import BaseModelfrom skyvern.exceptions import UrlGenerationFailurefrom skyvern.forge import appfrom skyvern.forge.prompts import prompt_engine+from skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContext-from skyvern.forge.sdk.schemas.observers import ObserverCruise, ObserverCruiseStatus, ObserverMetadata+from skyvern.forge.sdk.schemas.observers import (+ ObserverCruise,+ ObserverCruiseStatus,+ ObserverMetadata,+ ObserverThoughtScenario,+ ObserverThoughtType,+)from skyvern.forge.sdk.schemas.organizations import Organizationfrom skyvern.forge.sdk.schemas.tasks import ProxyLocationfrom skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunTimeline, WorkflowRunTimelineType@@ -76,6 +83,13 @@ async def initialize_observer_cruise(organization_id=organization.organization_id,)+ observer_thought = await app.DATABASE.create_observer_thought(+ observer_cruise_id=observer_cruise.observer_cruise_id,+ organization_id=organization.organization_id,+ observer_thought_type=ObserverThoughtType.metadata,+ observer_thought_scenario=ObserverThoughtScenario.generate_metadata,+ )+metadata_prompt = prompt_engine.load_prompt("observer_generate_metadata", user_goal=user_prompt, user_url=user_url)metadata_response = await app.SECONDARY_LLM_API_HANDLER(prompt=metadata_prompt, observer_cruise=observer_cruise)# validate@@ -103,6 +117,16 @@ async def initialize_observer_cruise(version=None,max_steps_override=max_steps_override,)+ await app.DATABASE.update_observer_thought(+ observer_thought_id=observer_thought.observer_thought_id,+ organization_id=organization.organization_id,+ workflow_run_id=workflow_run.workflow_run_id,+ workflow_id=new_workflow.workflow_id,+ workflow_permanent_id=new_workflow.workflow_permanent_id,+ thought=metadata_response.get("thoughts", ""),+ output=metadata.model_dump(),+ )+# update oserver cruiseobserver_cruise = await app.DATABASE.update_observer_cruise(observer_cruise_id=observer_cruise.observer_cruise_id,@@ -237,6 +261,8 @@ async def run_observer_cruise(workflow_run_id=workflow_run.workflow_run_id,workflow_id=workflow.workflow_id,workflow_permanent_id=workflow.workflow_permanent_id,+ observer_thought_type=ObserverThoughtType.plan,+ observer_thought_scenario=ObserverThoughtScenario.generate_plan,)observer_response = await app.LLM_API_HANDLER(prompt=observer_prompt,@@ -255,6 +281,7 @@ async def run_observer_cruise(observation = observer_response.get("page_info", "")thoughts: str = observer_response.get("thoughts", "")plan: str = observer_response.get("plan", "")+ task_type: str = observer_response.get("task_type", "")# Create and save observer thoughtawait app.DATABASE.update_observer_thought(observer_thought_id=observer_thought.observer_thought_id,@@ -262,6 +289,7 @@ async def run_observer_cruise(thought=thoughts,observation=observation,answer=plan,+ output={"task_type": task_type, "user_goal_achieved": user_goal_achieved},)if user_goal_achieved is True:@@ -274,7 +302,6 @@ async def run_observer_cruise(break# parse observer repsonse and run the next task- task_type = observer_response.get("task_type")if not task_type:LOG.error("No task type found in observer response", observer_response=observer_response)await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed(@@ -288,6 +315,8 @@ async def run_observer_cruise(block, block_yaml_list, parameter_yaml_list = await _generate_extraction_task(observer_cruise=observer_cruise,workflow_id=workflow_id,+ workflow_permanent_id=workflow.workflow_permanent_id,+ workflow_run_id=workflow_run_id,current_url=current_url,element_tree_in_prompt=element_tree_in_prompt,data_extraction_goal=plan,@@ -298,6 +327,8 @@ async def run_observer_cruise(original_url = url if i == 0 else Noneblock, block_yaml_list, parameter_yaml_list = await _generate_navigation_task(workflow_id=workflow_id,+ workflow_permanent_id=workflow.workflow_permanent_id,+ workflow_run_id=workflow_run_id,original_url=original_url,navigation_goal=plan,)@@ -307,6 +338,7 @@ async def run_observer_cruise(block, block_yaml_list, parameter_yaml_list, extraction_obj, inner_task = await _generate_loop_task(observer_cruise=observer_cruise,workflow_id=workflow_id,+ workflow_permanent_id=workflow.workflow_permanent_id,workflow_run_id=workflow_run_id,plan=plan,browser_state=browser_state,@@ -378,8 +410,18 @@ async def run_observer_cruise(task_history=task_history,local_datetime=datetime.now(context.tz_info).isoformat(),)+ observer_thought = await app.DATABASE.create_observer_thought(+ observer_cruise_id=observer_cruise_id,+ organization_id=organization_id,+ workflow_run_id=workflow_run_id,+ workflow_id=workflow_id,+ workflow_permanent_id=workflow.workflow_permanent_id,+ observer_thought_type=ObserverThoughtType.user_goal_check,+ observer_thought_scenario=ObserverThoughtScenario.user_goal_check,+ )completion_resp = await app.LLM_API_HANDLER(- prompt=observer_completion_prompt, observer_cruise=observer_cruise+ prompt=observer_completion_prompt,+ observer_cruise=observer_thought,)LOG.info("Observer completion check response",@@ -388,7 +430,15 @@ async def run_observer_cruise(workflow_run_id=workflow_run_id,task_history=task_history,)- if completion_resp.get("user_goal_achieved", False):+ user_goal_achieved = completion_resp.get("user_goal_achieved", False)+ thought = completion_resp.get("thoughts", "")+ await app.DATABASE.update_observer_thought(+ observer_thought_id=observer_thought.observer_thought_id,+ organization_id=organization_id,+ thought=thought,+ output={"user_goal_achieved": user_goal_achieved},+ )+ if user_goal_achieved:LOG.info("User goal achieved according to the observer completion check",iteration=i,@@ -514,6 +564,7 @@ async def _set_up_workflow_context(workflow_id: str, workflow_run_id: str) -> Noasync def _generate_loop_task(observer_cruise: ObserverCruise,workflow_id: str,+ workflow_permanent_id: str,workflow_run_id: str,plan: str,browser_state: BrowserState,@@ -525,7 +576,25 @@ async def _generate_loop_task("observer_loop_task_extraction_goal",plan=plan,)-+ data_extraction_thought = f"Going to generate a list of values to go through based on the plan: {plan}."+ observer_thought = await app.DATABASE.create_observer_thought(+ observer_cruise_id=observer_cruise.observer_cruise_id,+ organization_id=observer_cruise.organization_id,+ workflow_run_id=workflow_run_id,+ workflow_id=workflow_id,+ workflow_permanent_id=workflow_permanent_id,+ observer_thought_type=ObserverThoughtType.plan,+ observer_thought_scenario=ObserverThoughtScenario.extract_loop_values,+ thought=data_extraction_thought,+ )+ # generate screenshot artifact for the observer thought+ if scraped_page.screenshots:+ for screenshot in scraped_page.screenshots:+ await app.ARTIFACT_MANAGER.create_observer_thought_artifact(+ observer_thought=observer_thought,+ artifact_type=ArtifactType.SCREENSHOT_LLM,+ data=screenshot,+ )label = f"extraction_task_for_loop_{_generate_random_string()}"extraction_block_yaml = ExtractionBlockYAML(label=label,@@ -576,6 +645,13 @@ async def _generate_loop_task()raise+ # update the observer thought+ await app.DATABASE.update_observer_thought(+ observer_thought_id=observer_thought.observer_thought_id,+ organization_id=observer_cruise.organization_id,+ output=output_value_obj.model_dump(),+ )+# create ContextParameter for the loop over pointer that ForLoopBlock needs.loop_for_context_parameter = ContextParameter(key="loop_values",@@ -627,15 +703,31 @@ async def _generate_loop_task(is_link=output_value_obj.is_loop_value_link,loop_values=output_value_obj.loop_values,)+ observer_thought_task_in_loop = await app.DATABASE.create_observer_thought(+ observer_cruise_id=observer_cruise.observer_cruise_id,+ organization_id=observer_cruise.organization_id,+ workflow_run_id=workflow_run_id,+ workflow_id=workflow_id,+ workflow_permanent_id=workflow_permanent_id,+ observer_thought_type=ObserverThoughtType.internal_plan,+ observer_thought_scenario=ObserverThoughtScenario.generate_task_in_loop,+ )task_in_loop_metadata_response = await app.LLM_API_HANDLER(task_in_loop_metadata_prompt,screenshots=scraped_page.screenshots,- observer_cruise=observer_cruise,+ observer_thought=observer_thought_task_in_loop,)LOG.info("Task in loop metadata response", task_in_loop_metadata_response=task_in_loop_metadata_response)navigation_goal = task_in_loop_metadata_response.get("navigation_goal")data_extraction_goal = task_in_loop_metadata_response.get("data_extraction_goal")data_extraction_schema = task_in_loop_metadata_response.get("data_schema")+ thought = task_in_loop_metadata_response.get("thoughts")+ await app.DATABASE.update_observer_thought(+ observer_thought_id=observer_thought_task_in_loop.observer_thought_id,+ organization_id=observer_cruise.organization_id,+ thought=thought,+ output=task_in_loop_metadata_response,+ )if data_extraction_goal and navigation_goal:navigation_goal = (navigation_goal@@ -699,6 +791,8 @@ async def _generate_loop_task(async def _generate_extraction_task(observer_cruise: ObserverCruise,workflow_id: str,+ workflow_permanent_id: str,+ workflow_run_id: str,current_url: str,element_tree_in_prompt: str,data_extraction_goal: str,@@ -753,6 +847,8 @@ async def _generate_extraction_task(async def _generate_navigation_task(workflow_id: str,+ workflow_permanent_id: str,+ workflow_run_id: str,navigation_goal: str,original_url: str | None = None,) -> tuple[NavigationBlock, list[BLOCK_YAML_TYPES], list[PARAMETER_YAML_TYPES]]: